From 3a0629828ad1683be317be3c9cc0d06621921fda Mon Sep 17 00:00:00 2001 From: labhvam5 <88420539+labhvam5@users.noreply.github.com> Date: Fri, 20 Oct 2023 12:37:07 +0530 Subject: [PATCH] Import custom fields rework(#367) * Revert "Revert "cost-centers import rewrite (#345)" (#361)" This reverts commit a9c57aed594602403c03d0554b1a16f84af4a64b. * expense_custom_field rework * adding test * adding test * adding test * adding test for code cov * adding test * lint fix * test for pre-save-mapping-trigger * fixing error handlign in pre-save trigger * migration script --- apps/fyle/views.py | 8 +- apps/mappings/imports/modules/base.py | 27 +- .../imports/modules/expense_custom_fields.py | 176 +++++ apps/mappings/imports/queues.py | 10 + apps/mappings/imports/schedules.py | 12 +- apps/mappings/imports/tasks.py | 13 +- apps/mappings/signals.py | 97 ++- apps/mappings/tasks.py | 190 ------ apps/sage_intacct/dependent_fields.py | 30 +- .../create-update-new-custom-fields-import.py | 31 + tests/test_fyle/test_views.py | 5 +- .../test_imports/test_modules/fixtures.py | 620 ++++++++++++++++++ .../test_imports/test_modules/test_base.py | 26 +- .../test_modules/test_categories.py | 2 +- .../test_modules/test_cost_centers.py | 2 +- .../test_expense_custom_fields.py | 134 ++++ .../test_modules/test_projects.py | 2 +- tests/test_mappings/test_signals.py | 101 ++- tests/test_mappings/test_tasks.py | 69 +- 19 files changed, 1211 insertions(+), 344 deletions(-) create mode 100644 scripts/python/create-update-new-custom-fields-import.py create mode 100644 tests/test_mappings/test_imports/test_modules/test_expense_custom_fields.py diff --git a/apps/fyle/views.py b/apps/fyle/views.py index 008123b1..ce2d3b6c 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -344,16 +344,14 @@ def post(self, request, *args, **kwargs): chain = Chain() for mapping_setting in mapping_settings: - if mapping_setting.source_field in ['PROJECT', 'COST_CENTER']: + if mapping_setting.source_field in ['PROJECT', 'COST_CENTER'] or mapping_setting.is_custom: chain.append( 'apps.mappings.imports.tasks.trigger_import_via_schedule', int(kwargs['workspace_id']), mapping_setting.destination_field, - mapping_setting.source_field + mapping_setting.source_field, + mapping_setting.is_custom ) - elif mapping_setting.is_custom: - chain.append('apps.mappings.tasks.async_auto_create_custom_field_mappings', - int(kwargs['workspace_id'])) if chain.length() > 0: chain.run() diff --git a/apps/mappings/imports/modules/base.py b/apps/mappings/imports/modules/base.py index 737c2e53..3cf77122 100644 --- a/apps/mappings/imports/modules/base.py +++ b/apps/mappings/imports/modules/base.py @@ -40,7 +40,7 @@ def __init__( self.sync_after = sync_after - def __get_platform_class(self, platform: PlatformConnector): + def get_platform_class(self, platform: PlatformConnector): """ Get the platform class :param platform: PlatformConnector object @@ -48,7 +48,7 @@ def __get_platform_class(self, platform: PlatformConnector): """ return getattr(platform, self.platform_class_name) - def __get_auto_sync_permission(self): + def get_auto_sync_permission(self): """ Get the auto sync permission :return: bool @@ -59,7 +59,7 @@ def __get_auto_sync_permission(self): return is_auto_sync_status_allowed - def __construct_attributes_filter(self, attribute_type: str, paginated_destination_attribute_values: List[str] = []): + def construct_attributes_filter(self, attribute_type: str, paginated_destination_attribute_values: List[str] = []): """ Construct the attributes filter :param attribute_type: attribute type @@ -71,7 +71,7 @@ def __construct_attributes_filter(self, attribute_type: str, paginated_destinati 'workspace_id': self.workspace_id } - if self.sync_after: + if self.sync_after and self.platform_class_name != 'expense_custom_fields': filters['updated_at__gte'] = self.sync_after if paginated_destination_attribute_values: @@ -193,8 +193,11 @@ def sync_expense_attributes(self, platform: PlatformConnector): Sync expense attributes :param platform: PlatformConnector object """ - platform_class = self.__get_platform_class(platform) - platform_class.sync(sync_after=self.sync_after if self.sync_after else None) + platform_class = self.get_platform_class(platform) + if self.platform_class_name == 'expense_custom_fields': + platform_class.sync() + else: + platform_class.sync(sync_after=self.sync_after if self.sync_after else None) def sync_destination_attributes(self, sageintacct_attribute_type: str): """ @@ -229,9 +232,9 @@ def construct_payload_and_import_to_fyle( """ Construct Payload and Import to fyle in Batches """ - is_auto_sync_status_allowed = self.__get_auto_sync_permission() + is_auto_sync_status_allowed = self.get_auto_sync_permission() - filters = self.__construct_attributes_filter(self.destination_field) + filters = self.construct_attributes_filter(self.destination_field) destination_attributes_count = DestinationAttribute.objects.filter(**filters).count() @@ -249,7 +252,7 @@ def construct_payload_and_import_to_fyle( import_log.save() destination_attributes_generator = self.get_destination_attributes_generator(destination_attributes_count, filters) - platform_class = self.__get_platform_class(platform) + platform_class = self.get_platform_class(platform) for paginated_destination_attributes, is_last_batch in destination_attributes_generator: fyle_payload = self.setup_fyle_payload_creation( @@ -301,7 +304,7 @@ def get_existing_fyle_attributes(self, paginated_destination_attribute_values: L :param paginated_destination_attribute_values: List of DestinationAttribute values :return: Map of attribute value to attribute source_id """ - filters = self.__construct_attributes_filter(self.source_field, paginated_destination_attribute_values) + filters = self.construct_attributes_filter(self.source_field, paginated_destination_attribute_values) existing_expense_attributes_values = ExpenseAttribute.objects.filter(**filters).values('value', 'source_id') # This is a map of attribute name to attribute source_id return {attribute['value'].lower(): attribute['source_id'] for attribute in existing_expense_attributes_values} @@ -314,7 +317,9 @@ def post_to_fyle_and_sync(self, fyle_payload: List[object], resource_class, is_l :param is_last_batch: bool :param import_log: ImportLog object """ - if fyle_payload: + if fyle_payload and self.platform_class_name == 'expense_custom_fields': + resource_class.post(fyle_payload) + elif fyle_payload: resource_class.post_bulk(fyle_payload) self.update_import_log_post_import(is_last_batch, import_log) diff --git a/apps/mappings/imports/modules/expense_custom_fields.py b/apps/mappings/imports/modules/expense_custom_fields.py index e69de29b..6a564d41 100644 --- a/apps/mappings/imports/modules/expense_custom_fields.py +++ b/apps/mappings/imports/modules/expense_custom_fields.py @@ -0,0 +1,176 @@ +import math +from datetime import datetime +from typing import List, Dict +from apps.mappings.imports.modules.base import Base +from fyle_accounting_mappings.models import ( + DestinationAttribute, + ExpenseAttribute +) +from apps.mappings.exceptions import handle_import_exceptions +from apps.mappings.models import ImportLog +from fyle_integrations_platform_connector import PlatformConnector +from apps.workspaces.models import FyleCredential +from apps.mappings.constants import FYLE_EXPENSE_SYSTEM_FIELDS + + +class ExpenseCustomField(Base): + """ + Class for ExepenseCustomField module + """ + def __init__(self, workspace_id: int, source_field: str, destination_field: str, sync_after: datetime): + super().__init__( + workspace_id=workspace_id, + source_field=source_field, + destination_field=destination_field, + platform_class_name='expense_custom_fields', + sync_after=sync_after + ) + + def trigger_import(self): + """ + Trigger import for ExepenseCustomField module + """ + self.check_import_log_and_start_import() + + def construct_custom_field_placeholder(self, source_placeholder: str, fyle_attribute: str, existing_attribute: Dict): + """ + Construct placeholder for custom field + :param source_placeholder: Placeholder from mapping settings + :param fyle_attribute: Fyle attribute + :param existing_attribute: Existing attribute + """ + new_placeholder = None + placeholder = None + + if existing_attribute: + placeholder = existing_attribute['placeholder'] if 'placeholder' in existing_attribute else None + + # Here is the explanation of what's happening in the if-else ladder below + # source_field is the field that's save in mapping settings, this field user may or may not fill in the custom field form + # placeholder is the field that's saved in the detail column of destination attributes + # fyle_attribute is what we're constructing when both of these fields would not be available + + if not (source_placeholder or placeholder): + # If source_placeholder and placeholder are both None, then we're creating adding a self constructed placeholder + new_placeholder = 'Select {0}'.format(fyle_attribute) + elif not source_placeholder and placeholder: + # If source_placeholder is None but placeholder is not, then we're choosing same place holder as 1 in detail section + new_placeholder = placeholder + elif source_placeholder and not placeholder: + # If source_placeholder is not None but placeholder is None, then we're choosing the placeholder as filled by user in form + new_placeholder = source_placeholder + else: + # Else, we're choosing the placeholder as filled by user in form or None + new_placeholder = source_placeholder + + return new_placeholder + + def construct_fyle_expense_custom_field_payload( + self, + sageintacct_attributes: List[DestinationAttribute], + platform: PlatformConnector, + source_placeholder: str = None + ): + """ + Construct payload for expense custom fields + :param sageintacct_attributes: List of destination attributes + :param platform: PlatformConnector object + :param source_placeholder: Placeholder from mapping settings + """ + fyle_expense_custom_field_options = [] + fyle_attribute = self.source_field + + [fyle_expense_custom_field_options.append(sageintacct_attribute.value) for sageintacct_attribute in sageintacct_attributes] + + if fyle_attribute.lower() not in FYLE_EXPENSE_SYSTEM_FIELDS: + existing_attribute = ExpenseAttribute.objects.filter( + attribute_type=fyle_attribute, workspace_id=self.workspace_id).values_list('detail', flat=True).first() + + custom_field_id = None + + if existing_attribute is not None: + custom_field_id = existing_attribute['custom_field_id'] + + fyle_attribute = fyle_attribute.replace('_', ' ').title() + placeholder = self.construct_custom_field_placeholder(source_placeholder, fyle_attribute, existing_attribute) + + expense_custom_field_payload = { + 'field_name': fyle_attribute, + 'type': 'SELECT', + 'is_enabled': True, + 'is_mandatory': False, + 'placeholder': placeholder, + 'options': fyle_expense_custom_field_options, + 'code': None + } + + if custom_field_id: + expense_field = platform.expense_custom_fields.get_by_id(custom_field_id) + expense_custom_field_payload['id'] = custom_field_id + expense_custom_field_payload['is_mandatory'] = expense_field['is_mandatory'] + + return expense_custom_field_payload + + def construct_payload_and_import_to_fyle( + self, + platform: PlatformConnector, + import_log: ImportLog, + source_placeholder: str = None + ): + """ + Construct Payload and Import to fyle in Batches + """ + filters = self.construct_attributes_filter(self.destination_field) + + destination_attributes_count = DestinationAttribute.objects.filter(**filters).count() + + # If there are no destination attributes, mark the import as complete + if destination_attributes_count == 0: + import_log.status = 'COMPLETE' + import_log.last_successful_run_at = datetime.now() + import_log.error_log = [] + import_log.total_batches_count = 0 + import_log.processed_batches_count = 0 + import_log.save() + return + else: + import_log.total_batches_count = math.ceil(destination_attributes_count/200) + import_log.save() + + destination_attributes_generator = self.get_destination_attributes_generator(destination_attributes_count, filters) + platform_class = self.get_platform_class(platform) + + for paginated_destination_attributes, is_last_batch in destination_attributes_generator: + fyle_payload = self.construct_fyle_expense_custom_field_payload( + paginated_destination_attributes, + platform, + source_placeholder + ) + + self.post_to_fyle_and_sync( + fyle_payload=fyle_payload, + resource_class=platform_class, + is_last_batch=is_last_batch, + import_log=import_log + ) + + @handle_import_exceptions + def import_destination_attribute_to_fyle(self, import_log: ImportLog): + """ + Import destiantion_attributes field to Fyle and Auto Create Mappings + :param import_log: ImportLog object + """ + + fyle_credentials = FyleCredential.objects.get(workspace_id=self.workspace_id) + platform = PlatformConnector(fyle_credentials=fyle_credentials) + + self.sync_destination_attributes(self.destination_field) + + self.construct_payload_and_import_to_fyle( + platform=platform, + import_log=import_log + ) + + self.sync_expense_attributes(platform) + + self.create_mappings() diff --git a/apps/mappings/imports/queues.py b/apps/mappings/imports/queues.py index 2e985937..7c1b04ed 100644 --- a/apps/mappings/imports/queues.py +++ b/apps/mappings/imports/queues.py @@ -8,6 +8,7 @@ def chain_import_fields_to_fyle(workspace_id): :param workspace_id: Workspace Id """ mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True) + custom_field_mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, is_custom=True, import_to_fyle=True) configuration = Configuration.objects.get(workspace_id=workspace_id) chain = Chain() @@ -34,5 +35,14 @@ def chain_import_fields_to_fyle(workspace_id): mapping_setting.source_field ) + for custom_fields_mapping_setting in custom_field_mapping_settings: + chain.append( + 'apps.mappings.imports.tasks.trigger_import_via_schedule', + workspace_id, + custom_fields_mapping_setting.destination_field, + custom_fields_mapping_setting.source_field, + True + ) + if chain.length() > 0: chain.run() diff --git a/apps/mappings/imports/schedules.py b/apps/mappings/imports/schedules.py index 35539620..3a7505a6 100644 --- a/apps/mappings/imports/schedules.py +++ b/apps/mappings/imports/schedules.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta from django_q.models import Schedule from apps.workspaces.models import Configuration from fyle_accounting_mappings.models import MappingSetting @@ -34,8 +34,14 @@ def schedule_or_delete_fyle_import_tasks(configuration: Configuration, mapping_s source_field__in=['CATEGORY', 'PROJECT', 'COST_CENTER'] ).count() - # If the import_fields_count is 0, delete the schedule - if import_fields_count == 0: + custom_field_import_fields_count = MappingSetting.objects.filter( + import_to_fyle=True, + workspace_id=configuration.workspace_id, + is_custom=True + ).count() + + # If the import fields count is 0, delete the schedule + if import_fields_count == 0 and custom_field_import_fields_count == 0: Schedule.objects.filter( func='apps.mappings.imports.queues.chain_import_fields_to_fyle', args='{}'.format(configuration.workspace_id) diff --git a/apps/mappings/imports/tasks.py b/apps/mappings/imports/tasks.py index 75ad8d4c..d3692389 100644 --- a/apps/mappings/imports/tasks.py +++ b/apps/mappings/imports/tasks.py @@ -2,6 +2,7 @@ from apps.mappings.imports.modules.projects import Project from apps.mappings.imports.modules.categories import Category from apps.mappings.imports.modules.cost_centers import CostCenter +from apps.mappings.imports.modules.expense_custom_fields import ExpenseCustomField SOURCE_FIELD_CLASS_MAP = { 'PROJECT': Project, @@ -9,7 +10,7 @@ 'COST_CENTER': CostCenter, } -def trigger_import_via_schedule(workspace_id: int, destination_field: str, source_field: str): +def trigger_import_via_schedule(workspace_id: int, destination_field: str, source_field: str, is_custom: bool = False): """ Trigger import via schedule :param workspace_id: Workspace id @@ -19,6 +20,10 @@ def trigger_import_via_schedule(workspace_id: int, destination_field: str, sourc import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type=source_field).first() sync_after = import_log.last_successful_run_at if import_log else None - module_class = SOURCE_FIELD_CLASS_MAP[source_field] - item = module_class(workspace_id, destination_field, sync_after) - item.trigger_import() + if is_custom: + item = ExpenseCustomField(workspace_id, source_field, destination_field, sync_after) + item.trigger_import() + else: + module_class = SOURCE_FIELD_CLASS_MAP[source_field] + item = module_class(workspace_id, destination_field, sync_after) + item.trigger_import() diff --git a/apps/mappings/signals.py b/apps/mappings/signals.py index 5a05dcf5..3285963c 100644 --- a/apps/mappings/signals.py +++ b/apps/mappings/signals.py @@ -3,7 +3,9 @@ """ import logging import json +import traceback from django.db.models import Q +from datetime import datetime, timedelta, timezone from rest_framework.exceptions import ValidationError @@ -11,18 +13,24 @@ from django.dispatch import receiver from django_q.tasks import async_task -from fyle_accounting_mappings.models import MappingSetting, Mapping, EmployeeMapping, CategoryMapping, DestinationAttribute +from fyle_accounting_mappings.models import ( + MappingSetting, + Mapping, + EmployeeMapping, + CategoryMapping, + DestinationAttribute +) from fyle.platform.exceptions import WrongParamsError -from apps.mappings.tasks import ( - schedule_fyle_attributes_creation, - upload_attributes_to_fyle -) from apps.workspaces.models import Configuration from apps.mappings.helpers import schedule_or_delete_fyle_import_tasks from apps.mappings.imports.schedules import schedule_or_delete_fyle_import_tasks as new_schedule_or_delete_fyle_import_tasks from apps.tasks.models import Error from apps.mappings.models import LocationEntityMapping +from apps.mappings.imports.modules.expense_custom_fields import ExpenseCustomField +from apps.mappings.models import ImportLog +from fyle_integrations_platform_connector import PlatformConnector +from apps.workspaces.models import FyleCredential logger = logging.getLogger(__name__) @@ -115,8 +123,7 @@ def run_post_mapping_settings_triggers(sender, instance: MappingSetting, **kwarg new_schedule_or_delete_fyle_import_tasks(configuration, instance) if instance.is_custom: - schedule_fyle_attributes_creation(int(instance.workspace_id)) - + new_schedule_or_delete_fyle_import_tasks(configuration, instance) @receiver(pre_save, sender=MappingSetting) def run_pre_mapping_settings_triggers(sender, instance: MappingSetting, **kwargs): @@ -132,31 +139,65 @@ def run_pre_mapping_settings_triggers(sender, instance: MappingSetting, **kwargs if instance.source_field not in default_attributes and instance.import_to_fyle: # TODO: sync intacct fields before we upload custom field try: - upload_attributes_to_fyle( - workspace_id=int(instance.workspace_id), - sageintacct_attribute_type=instance.destination_field, - fyle_attribute_type=instance.source_field, - source_placeholder=instance.source_placeholder + workspace_id = int(instance.workspace_id) + # Checking is import_log exists or not if not create one + import_log, is_created = ImportLog.objects.get_or_create( + workspace_id=workspace_id, + attribute_type=instance.source_field, + defaults={ + 'status': 'IN_PROGRESS' + } + ) + + last_successful_run_at = None + if import_log and not is_created: + last_successful_run_at = import_log.last_successful_run_at if import_log.last_successful_run_at else None + time_difference = datetime.now() - timedelta(minutes=32) + offset_aware_time_difference = time_difference.replace(tzinfo=timezone.utc) + + # if the import_log is present and the last_successful_run_at is less than 30mins then we need to update it + # so that the schedule can run + if last_successful_run_at and offset_aware_time_difference\ + and (offset_aware_time_difference < last_successful_run_at): + import_log.last_successful_run_at = offset_aware_time_difference + last_successful_run_at = offset_aware_time_difference + import_log.save() + + # Creating the expense_custom_field object with the correct last_successful_run_at value + expense_custom_field = ExpenseCustomField( + workspace_id=workspace_id, + source_field=instance.source_field, + destination_field=instance.destination_field, + sync_after=last_successful_run_at ) + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) + platform = PlatformConnector(fyle_credentials=fyle_credentials) + + # setting the import_log status to IN_PROGRESS + import_log.status = 'IN_PROGRESS' + import_log.save() + + expense_custom_field.construct_payload_and_import_to_fyle(platform, import_log) + expense_custom_field.sync_expense_attributes(platform) + + # NOTE: We are not setting the import_log status to COMPLETE + # since the post_save trigger will run the import again in async manner + except WrongParamsError as error: logger.error( 'Error while creating %s workspace_id - %s in Fyle %s %s', instance.source_field, instance.workspace_id, error.message, {'error': error.response} ) - if error.response: - response = json.loads(error.response) - if response and 'message' in response and \ - response['message'] == ('duplicate key value violates unique constraint ' - '"idx_expense_fields_org_id_field_name_is_enabled_is_custom"'): - raise ValidationError({ - 'message': 'Duplicate custom field name', - 'field_name': instance.source_field - }) - - async_task( - 'apps.mappings.tasks.auto_create_expense_fields_mappings', - int(instance.workspace_id), - instance.destination_field, - instance.source_field - ) + if error.response and 'message' in error.response: + raise ValidationError({ + 'message': error.response['message'], + 'field_name': instance.source_field + }) + + # setting the import_log.last_successful_run_at to -30mins for the post_save_trigger + import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type=instance.source_field).first() + if import_log.last_successful_run_at: + last_successful_run_at = import_log.last_successful_run_at - timedelta(minutes=30) + import_log.last_successful_run_at = last_successful_run_at + import_log.save() diff --git a/apps/mappings/tasks.py b/apps/mappings/tasks.py index c0d8fda7..7feab5d5 100644 --- a/apps/mappings/tasks.py +++ b/apps/mappings/tasks.py @@ -248,196 +248,6 @@ def sync_sage_intacct_attributes(sageintacct_attribute_type: str, workspace_id: sage_intacct_connection.sync_user_defined_dimensions() -def construct_custom_field_placeholder(source_placeholder: str, fyle_attribute: str, existing_attribute: Dict): - new_placeholder = None - placeholder = None - - if existing_attribute: - placeholder = existing_attribute['placeholder'] if 'placeholder' in existing_attribute else None - - # Here is the explanation of what's happening in the if-else ladder below - # source_field is the field that's save in mapping settings, this field user may or may not fill in the custom field form - # placeholder is the field that's saved in the detail column of destination attributes - # fyle_attribute is what we're constructing when both of these fields would not be available - - if not (source_placeholder or placeholder): - # If source_placeholder and placeholder are both None, then we're creating adding a self constructed placeholder - new_placeholder = 'Select {0}'.format(fyle_attribute) - elif not source_placeholder and placeholder: - # If source_placeholder is None but placeholder is not, then we're choosing same place holder as 1 in detail section - new_placeholder = placeholder - elif source_placeholder and not placeholder: - # If source_placeholder is not None but placeholder is None, then we're choosing the placeholder as filled by user in form - new_placeholder = source_placeholder - else: - # Else, we're choosing the placeholder as filled by user in form or None - new_placeholder = source_placeholder - - return new_placeholder - - -def create_fyle_expense_custom_field_payload(sageintacct_attributes: List[DestinationAttribute], workspace_id: int, - fyle_attribute: str, platform: PlatformConnector, source_placeholder: str = None): - """ - Create Fyle Expense Custom Field Payload from SageIntacct Objects - :param workspace_id: Workspace ID - :param sageintacct_attributes: SageIntacct Objects - :param fyle_attribute: Fyle Attribute - :return: Fyle Expense Custom Field Payload - """ - - fyle_expense_custom_field_options = [] - - [fyle_expense_custom_field_options.append(sageintacct_attribute.value) for sageintacct_attribute in sageintacct_attributes] - - if fyle_attribute.lower() not in FYLE_EXPENSE_SYSTEM_FIELDS: - existing_attribute = ExpenseAttribute.objects.filter( - attribute_type=fyle_attribute, workspace_id=workspace_id).values_list('detail', flat=True).first() - - custom_field_id = None - - if existing_attribute is not None: - custom_field_id = existing_attribute['custom_field_id'] - - fyle_attribute = fyle_attribute.replace('_', ' ').title() - placeholder = construct_custom_field_placeholder(source_placeholder, fyle_attribute, existing_attribute) - - expense_custom_field_payload = { - 'field_name': fyle_attribute, - 'type': 'SELECT', - 'is_enabled': True, - 'is_mandatory': False, - 'placeholder': placeholder, - 'options': fyle_expense_custom_field_options, - 'code': None - } - - - if custom_field_id: - expense_field = platform.expense_custom_fields.get_by_id(custom_field_id) - expense_custom_field_payload['id'] = custom_field_id - expense_custom_field_payload['is_mandatory'] = expense_field['is_mandatory'] - - return expense_custom_field_payload - - -def upload_attributes_to_fyle(workspace_id: int, sageintacct_attribute_type: str, fyle_attribute_type: str, source_placeholder: str = None): - """ - Upload attributes to Fyle - """ - - fyle_credentials: FyleCredential = FyleCredential.objects.get(workspace_id=workspace_id) - - platform = PlatformConnector(fyle_credentials=fyle_credentials) - - sageintacct_attributes: List[DestinationAttribute] = DestinationAttribute.objects.filter( - workspace_id=workspace_id, attribute_type=sageintacct_attribute_type - ) - - sageintacct_attributes = remove_duplicates(sageintacct_attributes) - - fyle_custom_field_payload = create_fyle_expense_custom_field_payload( - fyle_attribute=fyle_attribute_type, - sageintacct_attributes=sageintacct_attributes, - workspace_id=workspace_id, - platform=platform, - source_placeholder=source_placeholder - ) - - if fyle_custom_field_payload: - platform.expense_custom_fields.post(fyle_custom_field_payload) - platform.expense_custom_fields.sync() - - return sageintacct_attributes - - -def auto_create_expense_fields_mappings( - workspace_id: int, sageintacct_attribute_type: str, fyle_attribute_type: str, source_placeholder: str = None -): - """ - Create Fyle Attributes Mappings - :return: mappings - """ - try: - fyle_attributes = None - fyle_attributes = upload_attributes_to_fyle( - workspace_id=workspace_id, - sageintacct_attribute_type=sageintacct_attribute_type, - fyle_attribute_type=fyle_attribute_type, - source_placeholder=source_placeholder - ) - - if fyle_attributes: - Mapping.bulk_create_mappings(fyle_attributes, fyle_attribute_type, sageintacct_attribute_type, workspace_id) - - except InvalidTokenError: - logger.info('Invalid Token or Invalid fyle credentials - %s', workspace_id) - - except FyleInvalidTokenError: - logger.info('Invalid Token for fyle') - - except InternalServerError: - logger.error('Internal server error while importing to Fyle') - - except WrongParamsError as exception: - logger.error( - 'Error while creating %s workspace_id - %s in Fyle %s %s', - fyle_attribute_type, workspace_id, exception.message, {'error': exception.response} - ) - except Exception: - error = traceback.format_exc() - error = { - 'error': error - } - logger.exception( - 'Error while creating %s workspace_id - %s error: %s', fyle_attribute_type, workspace_id, error - ) - - -def async_auto_create_custom_field_mappings(workspace_id: str): - mapping_settings = MappingSetting.objects.filter( - is_custom=True, import_to_fyle=True, workspace_id=workspace_id - ).all() - - for mapping_setting in mapping_settings: - try: - if mapping_setting.import_to_fyle: - sync_sage_intacct_attributes(mapping_setting.destination_field, workspace_id) - auto_create_expense_fields_mappings( - workspace_id, mapping_setting.destination_field, mapping_setting.source_field, - mapping_setting.source_placeholder - ) - except (SageIntacctCredential.DoesNotExist, InvalidTokenError): - logger.info('Invalid Token or Sage Intacct credentials does not exist - %s', workspace_id) - except NoPrivilegeError: - logger.info('Insufficient permission to access the requested module') - - -def schedule_fyle_attributes_creation(workspace_id: int): - mapping_settings = MappingSetting.objects.filter( - is_custom=True, import_to_fyle=True, workspace_id=workspace_id - ).all() - - if mapping_settings: - schedule, _= Schedule.objects.get_or_create( - func='apps.mappings.tasks.async_auto_create_custom_field_mappings', - args='{0}'.format(workspace_id), - defaults={ - 'schedule_type': Schedule.MINUTES, - 'minutes': 24 * 60, - 'next_run': datetime.now() + timedelta(hours=24) - } - ) - else: - schedule: Schedule = Schedule.objects.filter( - func='apps.mappings.tasks.async_auto_create_custom_field_mappings', - args=workspace_id - ).first() - - if schedule: - schedule.delete() - - def construct_filter_based_on_destination(reimbursable_destination_type: str): """ Construct Filter Based on Destination diff --git a/apps/sage_intacct/dependent_fields.py b/apps/sage_intacct/dependent_fields.py index 90328354..6186b621 100644 --- a/apps/sage_intacct/dependent_fields.py +++ b/apps/sage_intacct/dependent_fields.py @@ -14,7 +14,7 @@ from apps.fyle.helpers import connect_to_platform from apps.fyle.models import DependentFieldSetting -from apps.mappings.tasks import construct_custom_field_placeholder, sync_sage_intacct_attributes +from apps.mappings.tasks import sync_sage_intacct_attributes from apps.sage_intacct.models import CostType from apps.workspaces.models import SageIntacctCredential @@ -23,6 +23,34 @@ logger.level = logging.INFO +def construct_custom_field_placeholder(source_placeholder: str, fyle_attribute: str, existing_attribute: Dict): + new_placeholder = None + placeholder = None + + if existing_attribute: + placeholder = existing_attribute['placeholder'] if 'placeholder' in existing_attribute else None + + # Here is the explanation of what's happening in the if-else ladder below + # source_field is the field that's save in mapping settings, this field user may or may not fill in the custom field form + # placeholder is the field that's saved in the detail column of destination attributes + # fyle_attribute is what we're constructing when both of these fields would not be available + + if not (source_placeholder or placeholder): + # If source_placeholder and placeholder are both None, then we're creating adding a self constructed placeholder + new_placeholder = 'Select {0}'.format(fyle_attribute) + elif not source_placeholder and placeholder: + # If source_placeholder is None but placeholder is not, then we're choosing same place holder as 1 in detail section + new_placeholder = placeholder + elif source_placeholder and not placeholder: + # If source_placeholder is not None but placeholder is None, then we're choosing the placeholder as filled by user in form + new_placeholder = source_placeholder + else: + # Else, we're choosing the placeholder as filled by user in form or None + new_placeholder = source_placeholder + + return new_placeholder + + def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict) -> List[str]: projects = CostType.objects.filter(**filters).values('project_name').annotate(tasks=ArrayAgg('task_name', distinct=True)) projects_from_cost_types = [project['project_name'] for project in projects] diff --git a/scripts/python/create-update-new-custom-fields-import.py b/scripts/python/create-update-new-custom-fields-import.py new file mode 100644 index 00000000..20c6e400 --- /dev/null +++ b/scripts/python/create-update-new-custom-fields-import.py @@ -0,0 +1,31 @@ +from django.db import transaction +from datetime import datetime +from django_q.models import Schedule +from apps.workspaces.models import Configuration +from fyle_accounting_mappings.models import MappingSetting +existing_import_enabled_schedules = Schedule.objects.filter( + func__in=['apps.mappings.tasks.async_auto_create_custom_field_mappings'] +).values('args') +try: + with transaction.atomic(): + for schedule in existing_import_enabled_schedules: + mapping_setting = MappingSetting.objects.filter(workspace_id=schedule['args'], import_to_fyle=True, is_custom=True).first() + if mapping_setting: + Schedule.objects.update_or_create( + func='apps.mappings.imports.queues.chain_import_fields_to_fyle', + args=schedule['args'], + defaults={ + 'schedule_type': Schedule.MINUTES, + 'minutes':24 * 60, + 'next_run':datetime.now() + } + ) + raise Exception("This is a sanity check") +except Exception as e: + print(e) + +# Delete all the schedules for custom fields via SQL after running this script +# rollback; +# begin; +# delete from django_q_schedule where func = 'apps.mappings.tasks.async_auto_create_custom_field_mappings'; +# commit; diff --git a/tests/test_fyle/test_views.py b/tests/test_fyle/test_views.py index 1a223afa..5b9fbc01 100644 --- a/tests/test_fyle/test_views.py +++ b/tests/test_fyle/test_views.py @@ -9,7 +9,6 @@ from apps.tasks.models import TaskLog from django.urls import reverse from unittest import mock -from apps.fyle.helpers import sync_dimensions def test_exportable_expense_group_view(api_client, test_connection): @@ -399,8 +398,8 @@ def test_fyle_refresh_dimension(api_client, test_connection, mocker): ) mocker.patch( - 'apps.mappings.signals.upload_attributes_to_fyle', - return_value = [] + 'fyle_integrations_platform_connector.apis.ExpenseCustomFields.post', + return_value=[] ) workspace_id = 1 diff --git a/tests/test_mappings/test_imports/test_modules/fixtures.py b/tests/test_mappings/test_imports/test_modules/fixtures.py index 41dc6ae1..4497660e 100644 --- a/tests/test_mappings/test_imports/test_modules/fixtures.py +++ b/tests/test_mappings/test_imports/test_modules/fixtures.py @@ -10110,4 +10110,624 @@ 'description':'Cost Center - Service Line 3, Id - 300' } ] +} +expense_custom_field_data = { + 'create_new_auto_create_expense_custom_fields_expense_attributes_0':[ + { + "count": 2, + "data": [ + { + "category_ids": [ + 259385 + ], + "code": None, + "column_name": "text_column3", + "created_at": "2023-10-10T11:06:12.906551+00:00", + "default_value": None, + "field_name": "Luke", + "id": 229506, + "is_custom": True, + "is_enabled": True, + "is_mandatory": False, + "options": [ + "Australia", + "Bangalore", + "Canada", + "Elimination - Global", + "Elimination - NA", + "Elimination - Sub", + "Holding Company", + "Inactive loc", + "India", + "Leaf Village", + "London", + "Mist Village", + "New South Wales", + "Prod test", + "Sand village", + "South Africa", + "Testing", + "Testing Ash", + "United Kingdom", + "USA 1", + "USA 2" + ], + "org_id": "orqjgyJ21uge", + "parent_field_id": None, + "placeholder": "Select Luke", + "seq": 1, + "type": "SELECT", + "updated_at": "2023-10-10T13:24:33.787371+00:00" + }, + { + "category_ids": [ + 259385, + ], + "code": None, + "column_name": "text_column4", + "created_at": "2023-10-10T11:07:08.534779+00:00", + "default_value": None, + "field_name": "Cube", + "id": 229507, + "is_custom": True, + "is_enabled": True, + "is_mandatory": False, + "options": [ + "Butter Cookies 1", + "Butter Cookies 2", + "Enterprise", + "Killua Class", + "Midsize Business", + "Naruto test 1", + "Serizawa test 1", + "Serizawa test 2", + "Service Line 1", + "Service Line 2", + "Service Line 3", + "Small Business" + ], + "org_id": "orqjgyJ21uge", + "parent_field_id": None, + "placeholder": "Select Cube", + "seq": 1, + "type": "SELECT", + "updated_at": "2023-10-10T13:24:33.787371+00:00" + } + ], + "offset": 0 + } + ], + 'create_new_auto_create_expense_custom_fields_expense_attributes_1':[ + { + "count": 2, + "data": [ + { + "category_ids": [ + 259385 + ], + "code": None, + "column_name": "text_column3", + "created_at": "2023-10-10T11:06:12.906551+00:00", + "default_value": None, + "field_name": "Luke", + "id": 229506, + "is_custom": True, + "is_enabled": True, + "is_mandatory": False, + "options": [ + "Australia", + "Bangalore", + "Canada", + "Elimination - Global", + "Elimination - NA", + "Elimination - Sub", + "Holding Company", + "Inactive loc", + "India", + "Leaf Village", + "London", + "Mist Village", + "New South Wales", + "Prod test", + "Sand village", + "South Africa", + "Testing", + "Testing Ash", + "United Kingdom", + "USA 1", + "USA 2", + "USA 3", + "USA 4" + ], + "org_id": "orqjgyJ21uge", + "parent_field_id": None, + "placeholder": "Select Luke", + "seq": 1, + "type": "SELECT", + "updated_at": "2023-10-10T13:24:33.787371+00:00" + }, + { + "category_ids": [ + 259385, + ], + "code": None, + "column_name": "text_column4", + "created_at": "2023-10-10T11:07:08.534779+00:00", + "default_value": None, + "field_name": "Cube", + "id": 229507, + "is_custom": True, + "is_enabled": True, + "is_mandatory": False, + "options": [ + "Butter Cookies 1", + "Butter Cookies 2", + "Enterprise", + "Killua Class", + "Midsize Business", + "Naruto test 1", + "Serizawa test 1", + "Serizawa test 2", + "Service Line 1", + "Service Line 2", + "Service Line 3", + "Small Business" + ], + "org_id": "orqjgyJ21uge", + "parent_field_id": None, + "placeholder": "Select Cube", + "seq": 1, + "type": "SELECT", + "updated_at": "2023-10-10T13:24:33.787371+00:00" + } + ], + "offset": 0 + } + ], + 'create_new_auto_create_expense_custom_fields_destination_attributes': [ + { + 'RECORDNO':'1', + 'LOCATIONID':'100', + 'NAME':'USA 1', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'16', + 'LOCATIONID':'loc-inactive', + 'NAME':'Inactive loc', + 'PARENTID':'100', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'21', + 'LOCATIONID':'007', + 'NAME':'Leaf Village', + 'PARENTID':'100', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'22', + 'LOCATIONID':'008', + 'NAME':'Sand village', + 'PARENTID':'100', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'23', + 'LOCATIONID':'009', + 'NAME':'Mist Village', + 'PARENTID':'100', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'2', + 'LOCATIONID':'200', + 'NAME':'USA 2', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'3', + 'LOCATIONID':'300', + 'NAME':'Holding Company', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'19', + 'LOCATIONID':'Testing Ash', + 'NAME':'Testing Ash', + 'PARENTID':'300', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'4', + 'LOCATIONID':'900', + 'NAME':'Elimination - NA', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'5', + 'LOCATIONID':'910', + 'NAME':'Elimination - Global', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'11', + 'LOCATIONID':'in', + 'NAME':'India', + 'PARENTID':'910', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'12', + 'LOCATIONID':'dupid', + 'NAME':'Bangalore', + 'PARENTID':'in', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'6', + 'LOCATIONID':'920', + 'NAME':'Elimination - Sub', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'GBP' + }, + { + 'RECORDNO':'7', + 'LOCATIONID':'400', + 'NAME':'Canada', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'CAD' + }, + { + 'RECORDNO':'8', + 'LOCATIONID':'500', + 'NAME':'United Kingdom', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'GBP' + }, + { + 'RECORDNO':'15', + 'LOCATIONID':'London', + 'NAME':'London', + 'PARENTID':'500', + 'STATUS':'active', + 'CURRENCY':'GBP' + }, + { + 'RECORDNO':'9', + 'LOCATIONID':'600', + 'NAME':'Australia', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'AUD' + }, + { + 'RECORDNO':'14', + 'LOCATIONID':'700-New South Wales', + 'NAME':'New South Wales', + 'PARENTID':'600', + 'STATUS':'active', + 'CURRENCY':'AUD' + }, + { + 'RECORDNO':'17', + 'LOCATIONID':'ddd', + 'NAME':'Testing', + 'PARENTID':'600', + 'STATUS':'active', + 'CURRENCY':'AUD' + }, + { + 'RECORDNO':'18', + 'LOCATIONID':'kmnss', + 'NAME':'Prod test', + 'PARENTID':'600', + 'STATUS':'active', + 'CURRENCY':'AUD' + }, + { + 'RECORDNO':'10', + 'LOCATIONID':'700', + 'NAME':'South Africa', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'ZAR' + } + ], + 'create_new_auto_create_expense_custom_fields_destination_attributes_subsequent_run': [ + { + 'RECORDNO':'1', + 'LOCATIONID':'100', + 'NAME':'USA 1', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'16', + 'LOCATIONID':'loc-inactive', + 'NAME':'Inactive loc', + 'PARENTID':'100', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'21', + 'LOCATIONID':'007', + 'NAME':'Leaf Village', + 'PARENTID':'100', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'22', + 'LOCATIONID':'008', + 'NAME':'Sand village', + 'PARENTID':'100', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'23', + 'LOCATIONID':'009', + 'NAME':'Mist Village', + 'PARENTID':'100', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'2', + 'LOCATIONID':'200', + 'NAME':'USA 2', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'29', + 'LOCATIONID':'2001', + 'NAME':'USA 3', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'30', + 'LOCATIONID':'2002', + 'NAME':'USA 4', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'3', + 'LOCATIONID':'300', + 'NAME':'Holding Company', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'19', + 'LOCATIONID':'Testing Ash', + 'NAME':'Testing Ash', + 'PARENTID':'300', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'4', + 'LOCATIONID':'900', + 'NAME':'Elimination - NA', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'5', + 'LOCATIONID':'910', + 'NAME':'Elimination - Global', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'11', + 'LOCATIONID':'in', + 'NAME':'India', + 'PARENTID':'910', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'12', + 'LOCATIONID':'dupid', + 'NAME':'Bangalore', + 'PARENTID':'in', + 'STATUS':'active', + 'CURRENCY':'USD' + }, + { + 'RECORDNO':'6', + 'LOCATIONID':'920', + 'NAME':'Elimination - Sub', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'GBP' + }, + { + 'RECORDNO':'7', + 'LOCATIONID':'400', + 'NAME':'Canada', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'CAD' + }, + { + 'RECORDNO':'8', + 'LOCATIONID':'500', + 'NAME':'United Kingdom', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'GBP' + }, + { + 'RECORDNO':'15', + 'LOCATIONID':'London', + 'NAME':'London', + 'PARENTID':'500', + 'STATUS':'active', + 'CURRENCY':'GBP' + }, + { + 'RECORDNO':'9', + 'LOCATIONID':'600', + 'NAME':'Australia', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'AUD' + }, + { + 'RECORDNO':'14', + 'LOCATIONID':'700-New South Wales', + 'NAME':'New South Wales', + 'PARENTID':'600', + 'STATUS':'active', + 'CURRENCY':'AUD' + }, + { + 'RECORDNO':'17', + 'LOCATIONID':'ddd', + 'NAME':'Testing', + 'PARENTID':'600', + 'STATUS':'active', + 'CURRENCY':'AUD' + }, + { + 'RECORDNO':'18', + 'LOCATIONID':'kmnss', + 'NAME':'Prod test', + 'PARENTID':'600', + 'STATUS':'active', + 'CURRENCY':'AUD' + }, + { + 'RECORDNO':'10', + 'LOCATIONID':'700', + 'NAME':'South Africa', + 'PARENTID':None, + 'STATUS':'active', + 'CURRENCY':'ZAR' + } + ], + 'create_new_auto_create_expense_custom_fields_get_by_id': { + 'category_ids':[ + 259385, + 259386, + 259387, + 259388, + 259389, + 259390, + 259391, + 259392, + 259393, + 259394, + 259395, + 259396, + 259397, + 259398, + 259399, + ], + 'code':None, + 'column_name':'text_column3', + 'created_at':'2023-10-10T11:06:12.906551+00:00', + 'default_value':None, + 'field_name':'Luke', + 'id':229506, + 'is_custom':True, + 'is_enabled':True, + 'is_mandatory':False, + 'options':[ + 'Australia', + 'Bangalore', + 'Canada', + 'Elimination - Global', + 'Elimination - NA', + 'Elimination - Sub', + 'Holding Company', + 'Inactive loc', + 'India', + 'Leaf Village', + 'London', + 'Mist Village', + 'New South Wales', + 'Prod test', + 'Sand village', + 'South Africa', + 'Testing', + 'Testing Ash', + 'United Kingdom', + 'USA 1', + 'USA 2' + ], + 'org_id':'orqjgyJ21uge', + 'parent_field_id':None, + 'placeholder':'Select Luke', + 'seq':1, + 'type':'SELECT', + 'updated_at':'2023-10-11T07:42:24.133074+00:00' + }, + 'create_fyle_expense_custom_fields_payload_create_new_case': { + 'field_name':'Luke', + 'type':'SELECT', + 'is_enabled':True, + 'is_mandatory':False, + 'placeholder':'Select Luke', + 'options':[ + 'Australia', + 'Bangalore', + 'Canada', + 'Elimination - Global', + 'Elimination - NA', + 'Elimination - Sub', + 'Holding Company', + 'Inactive loc', + 'India', + 'Leaf Village', + 'London', + 'Mist Village', + 'New South Wales', + 'Prod test', + 'Sand village', + 'South Africa', + 'Testing', + 'Testing Ash', + 'United Kingdom', + 'USA 1', + 'USA 2' + ], + 'code':None, + 'id':229506 + }, + 'lol': {'field_name': 'Luke', 'type': 'SELECT', 'is_enabled': True, 'is_mandatory': False, 'placeholder': 'Select Luke', 'options': ['Australia', 'New South Wales'], 'code': None} } \ No newline at end of file diff --git a/tests/test_mappings/test_imports/test_modules/test_base.py b/tests/test_mappings/test_imports/test_modules/test_base.py index 3a0bfc5f..7683fc95 100644 --- a/tests/test_mappings/test_imports/test_modules/test_base.py +++ b/tests/test_mappings/test_imports/test_modules/test_base.py @@ -95,35 +95,35 @@ def test_remove_duplicates(db): attributes = base.remove_duplicate_attributes(attributes) assert len(attributes) == 55 -def test__get_platform_class(db): +def test_get_platform_class(db): base = get_base_class_instance() platform = get_platform_connection(1) - assert base._Base__get_platform_class(platform) == platform.projects + assert base.get_platform_class(platform) == platform.projects base = get_base_class_instance(workspace_id=1, source_field='CATEGORY', destination_field='ACCOUNT', platform_class_name='categories') - assert base._Base__get_platform_class(platform) == platform.categories + assert base.get_platform_class(platform) == platform.categories base = get_base_class_instance(workspace_id=1, source_field='COST_CENTER', destination_field='DEPARTMENT', platform_class_name='cost_centers') - assert base._Base__get_platform_class(platform) == platform.cost_centers + assert base.get_platform_class(platform) == platform.cost_centers -def test__get_auto_sync_permission(db): +def test_get_auto_sync_permission(db): base = get_base_class_instance() - assert base._Base__get_auto_sync_permission() == True + assert base.get_auto_sync_permission() == True base = get_base_class_instance(workspace_id=1, source_field='CATEGORY', destination_field='ACCOUNT', platform_class_name='categories') - assert base._Base__get_auto_sync_permission() == True + assert base.get_auto_sync_permission() == True base = get_base_class_instance(workspace_id=1, source_field='COST_CENTER', destination_field='DEPARTMENT', platform_class_name='cost_centers') - assert base._Base__get_auto_sync_permission() == False + assert base.get_auto_sync_permission() == False -def test__construct_attributes_filter(db): +def test_construct_attributes_filter(db): base = get_base_class_instance() - assert base._Base__construct_attributes_filter('PROJECT') == {'attribute_type': 'PROJECT', 'workspace_id': 1} + assert base.construct_attributes_filter('PROJECT') == {'attribute_type': 'PROJECT', 'workspace_id': 1} date_string = '2023-08-06 12:50:05.875029' sync_after = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f') @@ -131,11 +131,11 @@ def test__construct_attributes_filter(db): base = get_base_class_instance(workspace_id=1, source_field='CATEGORY', destination_field='ACCOUNT', platform_class_name='categories', sync_after=sync_after) - assert base._Base__construct_attributes_filter('CATEGORY') == {'attribute_type': 'CATEGORY', 'workspace_id': 1, 'updated_at__gte': sync_after} + assert base.construct_attributes_filter('CATEGORY') == {'attribute_type': 'CATEGORY', 'workspace_id': 1, 'updated_at__gte': sync_after} paginated_destination_attribute_values = ['Mobile App Redesign', 'Platform APIs', 'Fyle NetSuite Integration', 'Fyle Sage Intacct Integration', 'Support Taxes', 'T&M Project with Five Tasks', 'Fixed Fee Project with Five Tasks', 'General Overhead', 'General Overhead-Current', 'Youtube proj', 'Integrations', 'Yujiro', 'Pickle'] - assert base._Base__construct_attributes_filter('COST_CENTER', paginated_destination_attribute_values) == {'attribute_type': 'COST_CENTER', 'workspace_id': 1, 'updated_at__gte': sync_after, 'value__in': paginated_destination_attribute_values} + assert base.construct_attributes_filter('COST_CENTER', paginated_destination_attribute_values) == {'attribute_type': 'COST_CENTER', 'workspace_id': 1, 'updated_at__gte': sync_after, 'value__in': paginated_destination_attribute_values} def test_auto_create_destination_attributes(mocker, db): project = Project(1, 'PROJECT', None) @@ -360,7 +360,7 @@ def test_expense_attributes_sync_after(db): paginated_expense_attribute_values.append(expense_attribute.value) - filters = project._Base__construct_attributes_filter('PROJECT', paginated_expense_attribute_values) + filters = project.construct_attributes_filter('PROJECT', paginated_expense_attribute_values) expense_attributes = ExpenseAttribute.objects.filter(**filters) diff --git a/tests/test_mappings/test_imports/test_modules/test_categories.py b/tests/test_mappings/test_imports/test_modules/test_categories.py index def2023d..92dd34ca 100644 --- a/tests/test_mappings/test_imports/test_modules/test_categories.py +++ b/tests/test_mappings/test_imports/test_modules/test_categories.py @@ -275,7 +275,7 @@ def test_construct_fyle_payload(db): # create new case paginated_destination_attributes = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='EXPENSE_TYPE') existing_fyle_attributes_map = {} - is_auto_sync_status_allowed = category._Base__get_auto_sync_permission() + is_auto_sync_status_allowed = category.get_auto_sync_permission() fyle_payload = category.construct_fyle_payload( paginated_destination_attributes, diff --git a/tests/test_mappings/test_imports/test_modules/test_cost_centers.py b/tests/test_mappings/test_imports/test_modules/test_cost_centers.py index 91b4740e..41d5b82e 100644 --- a/tests/test_mappings/test_imports/test_modules/test_cost_centers.py +++ b/tests/test_mappings/test_imports/test_modules/test_cost_centers.py @@ -131,7 +131,7 @@ def test_construct_fyle_payload(db): # create new case paginated_destination_attributes = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='CLASS') existing_fyle_attributes_map = {} - is_auto_sync_status_allowed = cost_center._Base__get_auto_sync_permission() + is_auto_sync_status_allowed = cost_center.get_auto_sync_permission() fyle_payload = cost_center.construct_fyle_payload( paginated_destination_attributes, diff --git a/tests/test_mappings/test_imports/test_modules/test_expense_custom_fields.py b/tests/test_mappings/test_imports/test_modules/test_expense_custom_fields.py new file mode 100644 index 00000000..21a2bb5e --- /dev/null +++ b/tests/test_mappings/test_imports/test_modules/test_expense_custom_fields.py @@ -0,0 +1,134 @@ +import pytest +from unittest import mock +from apps.mappings.imports.modules.expense_custom_fields import ExpenseCustomField +from fyle_accounting_mappings.models import ( + DestinationAttribute, + ExpenseAttribute, + Mapping +) +from fyle_integrations_platform_connector import PlatformConnector +from apps.workspaces.models import FyleCredential +from .fixtures import expense_custom_field_data +from .helpers import get_platform_connection + +def test_sync_expense_atrributes(mocker, db): + workspace_id = 1 + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) + platform = PlatformConnector(fyle_credentials=fyle_credentials) + + expense_attribute_count = ExpenseAttribute.objects.filter(workspace_id=workspace_id, attribute_type='LUKE').count() + assert expense_attribute_count == 0 + + mocker.patch( + 'fyle.platform.apis.v1beta.admin.expense_fields.list_all', + return_value=[] + ) + + expense_custom_field = ExpenseCustomField(workspace_id, 'LUKE', 'CLASS', None) + expense_custom_field.sync_expense_attributes(platform) + + expense_attribute_count = ExpenseAttribute.objects.filter(workspace_id=workspace_id, attribute_type='LUKE').count() + assert expense_attribute_count == 0 + + mocker.patch( + 'fyle.platform.apis.v1beta.admin.expense_fields.list_all', + return_value=expense_custom_field_data['create_new_auto_create_expense_custom_fields_expense_attributes_0'] + ) + + expense_custom_field.sync_expense_attributes(platform) + + expense_attribute_count = ExpenseAttribute.objects.filter(workspace_id=workspace_id, attribute_type='LUKE').count() + assert expense_attribute_count == 21 + +def test_auto_create_destination_attributes(mocker, db): + expense_custom_field = ExpenseCustomField(1, 'LUKE', 'LOCATION', None) + expense_custom_field.sync_after = None + + # delete all destination attributes, expense attributes and mappings + Mapping.objects.filter(workspace_id=1, source_type='LUKE', destination_type='LOCATION').delete() + DestinationAttribute.objects.filter(workspace_id=1, attribute_type='LOCATION').delete() + ExpenseAttribute.objects.filter(workspace_id=1, attribute_type='LUKE').delete() + + # create new case for projects import + with mock.patch('fyle.platform.apis.v1beta.admin.expense_fields.list_all') as mock_call: + mocker.patch( + 'fyle_integrations_platform_connector.apis.ExpenseCustomFields.post', + return_value=[] + ) + mocker.patch( + 'sageintacctsdk.apis.Locations.get_all', + return_value=expense_custom_field_data['create_new_auto_create_expense_custom_fields_destination_attributes'] + ) + mock_call.side_effect = [ + expense_custom_field_data['create_new_auto_create_expense_custom_fields_expense_attributes_0'], + ] + + expense_attributes_count = ExpenseAttribute.objects.filter(workspace_id=1, attribute_type = 'LUKE').count() + + assert expense_attributes_count == 0 + + mappings_count = Mapping.objects.filter(workspace_id=1, source_type='LUKE', destination_type='LOCATION').count() + + assert mappings_count == 0 + + expense_custom_field.trigger_import() + + expense_attributes_count = ExpenseAttribute.objects.filter(workspace_id=1, attribute_type = 'LUKE').count() + + assert expense_attributes_count == 21 + + mappings_count = Mapping.objects.filter(workspace_id=1, source_type='LUKE', destination_type='LOCATION').count() + + assert mappings_count == 21 + + # create new expense_custom_field mapping for sub-sequent run (we will be adding 2 new LOCATION) + with mock.patch('fyle.platform.apis.v1beta.admin.expense_fields.list_all') as mock_call: + mocker.patch( + 'fyle_integrations_platform_connector.apis.ExpenseCustomFields.post', + return_value=[] + ) + mocker.patch( + 'fyle_integrations_platform_connector.apis.ExpenseCustomFields.get_by_id', + return_value=expense_custom_field_data['create_new_auto_create_expense_custom_fields_get_by_id'] + ) + mocker.patch( + 'sageintacctsdk.apis.Locations.get_all', + return_value=expense_custom_field_data['create_new_auto_create_expense_custom_fields_destination_attributes_subsequent_run'] + ) + mock_call.side_effect = [ + expense_custom_field_data['create_new_auto_create_expense_custom_fields_expense_attributes_1'], + ] + + expense_attributes_count = ExpenseAttribute.objects.filter(workspace_id=1, attribute_type = 'LUKE').count() + + assert expense_attributes_count == 21 + + mappings_count = Mapping.objects.filter(workspace_id=1, source_type='LUKE', destination_type='LOCATION').count() + + assert mappings_count == 21 + + expense_custom_field.trigger_import() + + expense_attributes_count = ExpenseAttribute.objects.filter(workspace_id=1, attribute_type = 'LUKE').count() + + assert expense_attributes_count == 21 + 2 + + mappings_count = Mapping.objects.filter(workspace_id=1, source_type='LUKE', destination_type='LOCATION').count() + + assert mappings_count == 21 + 2 + + +def test_construct_fyle_expense_custom_field_payload(db): + expense_custom_field = ExpenseCustomField(1, 'LUKE', 'LOCATION', None) + expense_custom_field.sync_after = None + platform = get_platform_connection(1) + + # create new case + paginated_destination_attributes = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='LOCATION') + + fyle_payload = expense_custom_field.construct_fyle_expense_custom_field_payload( + paginated_destination_attributes, + platform + ) + + assert fyle_payload == expense_custom_field_data['lol'] diff --git a/tests/test_mappings/test_imports/test_modules/test_projects.py b/tests/test_mappings/test_imports/test_modules/test_projects.py index 1f031f04..680b2669 100644 --- a/tests/test_mappings/test_imports/test_modules/test_projects.py +++ b/tests/test_mappings/test_imports/test_modules/test_projects.py @@ -10,7 +10,7 @@ def test_construct_fyle_payload(db): # create new case paginated_destination_attributes = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='PROJECT') existing_fyle_attributes_map = {} - is_auto_sync_status_allowed = project._Base__get_auto_sync_permission() + is_auto_sync_status_allowed = project.get_auto_sync_permission() fyle_payload = project.construct_fyle_payload( paginated_destination_attributes, diff --git a/tests/test_mappings/test_signals.py b/tests/test_mappings/test_signals.py index 276b59b4..9cfc17c6 100644 --- a/tests/test_mappings/test_signals.py +++ b/tests/test_mappings/test_signals.py @@ -1,12 +1,21 @@ from asyncio.log import logger +from datetime import datetime, timedelta, timezone import pytest import json from unittest import mock +from django.db import transaction from django_q.models import Schedule -from fyle_accounting_mappings.models import MappingSetting, Mapping, ExpenseAttribute, EmployeeMapping, CategoryMapping +from fyle_accounting_mappings.models import ( + MappingSetting, + Mapping, + ExpenseAttribute, + EmployeeMapping, + CategoryMapping, + DestinationAttribute +) from apps.tasks.models import Error from apps.workspaces.models import Configuration, Workspace -from apps.mappings.models import LocationEntityMapping +from apps.mappings.models import LocationEntityMapping, ImportLog from fyle.platform.exceptions import WrongParamsError from ..test_fyle.fixtures import data as fyle_data @@ -198,11 +207,11 @@ def test_run_post_mapping_settings_triggers(db, mocker, test_connection): mapping_setting.save() schedule = Schedule.objects.filter( - func='apps.mappings.tasks.async_auto_create_custom_field_mappings', + func='apps.mappings.imports.queues.chain_import_fields_to_fyle', args='{}'.format(workspace_id), ).first() - assert schedule.func == 'apps.mappings.tasks.async_auto_create_custom_field_mappings' + assert schedule.func == 'apps.mappings.imports.queues.chain_import_fields_to_fyle' assert schedule.args == '1' @@ -244,25 +253,87 @@ def test_run_pre_mapping_settings_triggers(db, mocker, test_connection): ) workspace_id = 1 + custom_mappings = Mapping.objects.filter(workspace_id=workspace_id, source_type='CUSTOM_INTENTS').count() + assert custom_mappings == 0 - custom_mappings = Mapping.objects.filter(workspace_id=workspace_id, source_type='CUSTOM_INTENTs').count() + try: + mapping_setting = MappingSetting.objects.create( + source_field='CUSTOM_INTENTS', + destination_field='CUSTOM_INTENTS', + workspace_id=workspace_id, + import_to_fyle=True, + is_custom=True + ) + except: + logger.info('Duplicate custom field name') + + custom_mappings = Mapping.objects.last() + + custom_mappings = Mapping.objects.filter(workspace_id=workspace_id, source_type='CUSTOM_INTENTS').count() assert custom_mappings == 0 - with mock.patch('apps.mappings.signals.upload_attributes_to_fyle') as mock_call: - mock_call.side_effect = WrongParamsError(msg='invalid params', response=json.dumps({'code': 400, 'message': 'duplicate key value violates unique constraint ' - '"idx_expense_fields_org_id_field_name_is_enabled_is_custom"', 'Detail': 'Invalid parametrs'})) + import_log = ImportLog.objects.filter( + workspace_id=1, + attribute_type='CUSTOM_INTENTS' + ).first() + + assert import_log.status == 'COMPLETE' + + time_difference = datetime.now() - timedelta(hours=2) + offset_aware_time_difference = time_difference.replace(tzinfo=timezone.utc) + import_log.last_successful_run_at = offset_aware_time_difference + import_log.save() + + ImportLog.objects.filter(workspace_id=1, attribute_type='CUSTOM_INTENTS').delete() + + # case where error will occur but we reach the case where there are no destination attributes + # so we mark the import as complete + with mock.patch('fyle_integrations_platform_connector.apis.ExpenseCustomFields.post') as mock_call: + mock_call.side_effect = WrongParamsError(msg='invalid params', response={'code': 400, 'message': 'duplicate key value violates unique constraint ' + '"idx_expense_fields_org_id_field_name_is_enabled_is_custom"', 'Detail': 'Invalid parametrs'}) + + mapping_setting = MappingSetting( + source_field='CUSTOM_INTENTS', + destination_field='CUSTOM_INTENTS', + workspace_id=workspace_id, + import_to_fyle=True, + is_custom=True + ) + + try: + with transaction.atomic(): + mapping_setting.save() + except: + logger.info('duplicate key value violates unique constraint') + + with mock.patch('fyle_integrations_platform_connector.apis.ExpenseCustomFields.post') as mock_call: + mock_call.side_effect = WrongParamsError(msg='invalid params', response={'data': None, 'error': 'InvalidUsage', 'message': 'text_column cannot be added as it exceeds the maximum limit(15) of columns of a single type'}) + mapping_setting = MappingSetting( - source_field='CUSTOM_INTENTs', - destination_field='CUSTOM_INTENTs', + source_field='CUSTOM_INTENTS', + destination_field='CUSTOM_INTENTS', workspace_id=workspace_id, import_to_fyle=True, is_custom=True ) + try: mapping_setting.save() except: - logger.info('Duplicate custom field name') - custom_mappings = Mapping.objects.last() - - custom_mappings = Mapping.objects.filter(workspace_id=workspace_id, source_type='CUSTOM_INTENTs').count() - assert custom_mappings == 0 + logger.info('text_column cannot be added as it exceeds the maximum limit(15) of columns of a single type') + + with mock.patch('fyle_integrations_platform_connector.apis.ExpenseCustomFields.post') as mock_call: + mock_call.side_effect = WrongParamsError(msg='invalid params', response={'data': None,'error': 'IntegrityError','message': 'The values ("or79Cob97KSh", "text_column15", "1") already exists'}) + + mapping_setting = MappingSetting( + source_field='CUSTOM_INTENTS', + destination_field='CUSTOM_INTENTS', + workspace_id=workspace_id, + import_to_fyle=True, + is_custom=True + ) + + try: + mapping_setting.save() + except: + logger.info('The values ("or79Cob97KSh", "text_column15", "1") already exists') diff --git a/tests/test_mappings/test_tasks.py b/tests/test_mappings/test_tasks.py index d0246bf3..38832aae 100644 --- a/tests/test_mappings/test_tasks.py +++ b/tests/test_mappings/test_tasks.py @@ -3,7 +3,6 @@ from fyle_accounting_mappings.models import ( DestinationAttribute, Mapping, - MappingSetting, EmployeeMapping, ExpenseAttribute ) @@ -216,73 +215,7 @@ def test_schedule_auto_map_employees(db): args='{}'.format(workspace_id), ).first() - assert schedule == None - - -def test_schedule_fyle_attributes_creation(db, mocker): - workspace_id = 1 - - mapping_setting = MappingSetting.objects.last() - mapping_setting.is_custom=True - mapping_setting.import_to_fyle=True - mapping_setting.save() - - schedule_fyle_attributes_creation(workspace_id) - - mocker.patch( - 'fyle_integrations_platform_connector.apis.ExpenseCustomFields.post', - return_value=[] - ) - - mocker.patch( - 'sageintacctsdk.apis.Dimensions.get_all', - return_value=intacct_data['get_dimensions'] - ) - - schedule = Schedule.objects.filter( - func='apps.mappings.tasks.async_auto_create_custom_field_mappings', - args='{}'.format(workspace_id), - ).first() - - assert schedule.func == 'apps.mappings.tasks.async_auto_create_custom_field_mappings' - - async_auto_create_custom_field_mappings(workspace_id) - - schedule_fyle_attributes_creation(2) - schedule = Schedule.objects.filter( - func='apps.mappings.tasks.async_auto_create_custom_field_mappings', - args='{}'.format(workspace_id), - ).first() - - assert schedule.func == 'apps.mappings.tasks.async_auto_create_custom_field_mappings' - - with mock.patch('apps.mappings.tasks.async_auto_create_custom_field_mappings') as mock_call: - mock_call.side_effect = NoPrivilegeError(msg='insufficient permission', response='insufficient permission') - async_auto_create_custom_field_mappings(workspace_id=workspace_id) - - -def test_auto_create_expense_fields_mappings(db, mocker, create_mapping_setting): - mocker.patch( - 'fyle_integrations_platform_connector.apis.ExpenseCustomFields.post', - return_value=[] - ) - mocker.patch( - 'fyle_integrations_platform_connector.apis.ExpenseCustomFields.sync', - return_value=[] - ) - workspace_id = 1 - - auto_create_expense_fields_mappings(workspace_id, 'TASK', 'COST_CODES', None) - mappings = Mapping.objects.filter(workspace_id=workspace_id, destination_type='TASK').count() - assert mappings == 0 - - auto_create_expense_fields_mappings(workspace_id, 'COST_CENTER', 'COST_CENTER', 'Select Cost Center') - - cost_center = DestinationAttribute.objects.filter(workspace_id=workspace_id, attribute_type='COST_CENTER').count() - mappings = Mapping.objects.filter(workspace_id=workspace_id, source_type='COST_CENTER').count() - - assert cost_center == 1 - assert mappings == 0 + assert schedule == None def test_sync_sage_intacct_attributes(mocker, db, create_dependent_field_setting, create_cost_type):