Skip to content

Commit

Permalink
Import custom re revert (#383)
Browse files Browse the repository at this point in the history
* Revert "Revert "cost-centers import rewrite (#345)" (#361)"

This reverts commit a9c57ae.

* Revert "Reverting expense fields (#380)"

This reverts commit 06313a4.
  • Loading branch information
labhvam5 authored Oct 31, 2023
1 parent 216fc5a commit cabff53
Show file tree
Hide file tree
Showing 19 changed files with 1,215 additions and 344 deletions.
8 changes: 3 additions & 5 deletions apps/fyle/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,16 +360,14 @@ def post(self, request, *args, **kwargs):
chain = Chain()

for mapping_setting in mapping_settings:
if mapping_setting.source_field in ['PROJECT', 'COST_CENTER']:
if mapping_setting.source_field in ['PROJECT', 'COST_CENTER'] or mapping_setting.is_custom:
chain.append(
'apps.mappings.imports.tasks.trigger_import_via_schedule',
int(kwargs['workspace_id']),
mapping_setting.destination_field,
mapping_setting.source_field
mapping_setting.source_field,
mapping_setting.is_custom
)
elif mapping_setting.is_custom:
chain.append('apps.mappings.tasks.async_auto_create_custom_field_mappings',
int(kwargs['workspace_id']))

if chain.length() > 0:
chain.run()
Expand Down
27 changes: 16 additions & 11 deletions apps/mappings/imports/modules/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ def __init__(
self.sync_after = sync_after


def __get_platform_class(self, platform: PlatformConnector):
def get_platform_class(self, platform: PlatformConnector):
"""
Get the platform class
:param platform: PlatformConnector object
:return: platform class
"""
return getattr(platform, self.platform_class_name)

def __get_auto_sync_permission(self):
def get_auto_sync_permission(self):
"""
Get the auto sync permission
:return: bool
Expand All @@ -59,7 +59,7 @@ def __get_auto_sync_permission(self):

return is_auto_sync_status_allowed

def __construct_attributes_filter(self, attribute_type: str, paginated_destination_attribute_values: List[str] = []):
def construct_attributes_filter(self, attribute_type: str, paginated_destination_attribute_values: List[str] = []):
"""
Construct the attributes filter
:param attribute_type: attribute type
Expand All @@ -71,7 +71,7 @@ def __construct_attributes_filter(self, attribute_type: str, paginated_destinati
'workspace_id': self.workspace_id
}

if self.sync_after:
if self.sync_after and self.platform_class_name != 'expense_custom_fields':
filters['updated_at__gte'] = self.sync_after

if paginated_destination_attribute_values:
Expand Down Expand Up @@ -193,8 +193,11 @@ def sync_expense_attributes(self, platform: PlatformConnector):
Sync expense attributes
:param platform: PlatformConnector object
"""
platform_class = self.__get_platform_class(platform)
platform_class.sync(sync_after=self.sync_after if self.sync_after else None)
platform_class = self.get_platform_class(platform)
if self.platform_class_name == 'expense_custom_fields':
platform_class.sync()
else:
platform_class.sync(sync_after=self.sync_after if self.sync_after else None)

def sync_destination_attributes(self, sageintacct_attribute_type: str):
"""
Expand Down Expand Up @@ -229,9 +232,9 @@ def construct_payload_and_import_to_fyle(
"""
Construct Payload and Import to fyle in Batches
"""
is_auto_sync_status_allowed = self.__get_auto_sync_permission()
is_auto_sync_status_allowed = self.get_auto_sync_permission()

filters = self.__construct_attributes_filter(self.destination_field)
filters = self.construct_attributes_filter(self.destination_field)

destination_attributes_count = DestinationAttribute.objects.filter(**filters).count()

Expand All @@ -249,7 +252,7 @@ def construct_payload_and_import_to_fyle(
import_log.save()

destination_attributes_generator = self.get_destination_attributes_generator(destination_attributes_count, filters)
platform_class = self.__get_platform_class(platform)
platform_class = self.get_platform_class(platform)

for paginated_destination_attributes, is_last_batch in destination_attributes_generator:
fyle_payload = self.setup_fyle_payload_creation(
Expand Down Expand Up @@ -301,7 +304,7 @@ def get_existing_fyle_attributes(self, paginated_destination_attribute_values: L
:param paginated_destination_attribute_values: List of DestinationAttribute values
:return: Map of attribute value to attribute source_id
"""
filters = self.__construct_attributes_filter(self.source_field, paginated_destination_attribute_values)
filters = self.construct_attributes_filter(self.source_field, paginated_destination_attribute_values)
existing_expense_attributes_values = ExpenseAttribute.objects.filter(**filters).values('value', 'source_id')
# This is a map of attribute name to attribute source_id
return {attribute['value'].lower(): attribute['source_id'] for attribute in existing_expense_attributes_values}
Expand All @@ -314,7 +317,9 @@ def post_to_fyle_and_sync(self, fyle_payload: List[object], resource_class, is_l
:param is_last_batch: bool
:param import_log: ImportLog object
"""
if fyle_payload:
if fyle_payload and self.platform_class_name == 'expense_custom_fields':
resource_class.post(fyle_payload)
elif fyle_payload:
resource_class.post_bulk(fyle_payload)

self.update_import_log_post_import(is_last_batch, import_log)
Expand Down
176 changes: 176 additions & 0 deletions apps/mappings/imports/modules/expense_custom_fields.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
import math
from datetime import datetime
from typing import List, Dict
from apps.mappings.imports.modules.base import Base
from fyle_accounting_mappings.models import (
DestinationAttribute,
ExpenseAttribute
)
from apps.mappings.exceptions import handle_import_exceptions
from apps.mappings.models import ImportLog
from fyle_integrations_platform_connector import PlatformConnector
from apps.workspaces.models import FyleCredential
from apps.mappings.constants import FYLE_EXPENSE_SYSTEM_FIELDS


class ExpenseCustomField(Base):
"""
Class for ExepenseCustomField module
"""
def __init__(self, workspace_id: int, source_field: str, destination_field: str, sync_after: datetime):
super().__init__(
workspace_id=workspace_id,
source_field=source_field,
destination_field=destination_field,
platform_class_name='expense_custom_fields',
sync_after=sync_after
)

def trigger_import(self):
"""
Trigger import for ExepenseCustomField module
"""
self.check_import_log_and_start_import()

def construct_custom_field_placeholder(self, source_placeholder: str, fyle_attribute: str, existing_attribute: Dict):
"""
Construct placeholder for custom field
:param source_placeholder: Placeholder from mapping settings
:param fyle_attribute: Fyle attribute
:param existing_attribute: Existing attribute
"""
new_placeholder = None
placeholder = None

if existing_attribute:
placeholder = existing_attribute['placeholder'] if 'placeholder' in existing_attribute else None

# Here is the explanation of what's happening in the if-else ladder below
# source_field is the field that's save in mapping settings, this field user may or may not fill in the custom field form
# placeholder is the field that's saved in the detail column of destination attributes
# fyle_attribute is what we're constructing when both of these fields would not be available

if not (source_placeholder or placeholder):
# If source_placeholder and placeholder are both None, then we're creating adding a self constructed placeholder
new_placeholder = 'Select {0}'.format(fyle_attribute)
elif not source_placeholder and placeholder:
# If source_placeholder is None but placeholder is not, then we're choosing same place holder as 1 in detail section
new_placeholder = placeholder
elif source_placeholder and not placeholder:
# If source_placeholder is not None but placeholder is None, then we're choosing the placeholder as filled by user in form
new_placeholder = source_placeholder
else:
# Else, we're choosing the placeholder as filled by user in form or None
new_placeholder = source_placeholder

return new_placeholder

def construct_fyle_expense_custom_field_payload(
self,
sageintacct_attributes: List[DestinationAttribute],
platform: PlatformConnector,
source_placeholder: str = None
):
"""
Construct payload for expense custom fields
:param sageintacct_attributes: List of destination attributes
:param platform: PlatformConnector object
:param source_placeholder: Placeholder from mapping settings
"""
fyle_expense_custom_field_options = []
fyle_attribute = self.source_field

[fyle_expense_custom_field_options.append(sageintacct_attribute.value) for sageintacct_attribute in sageintacct_attributes]

if fyle_attribute.lower() not in FYLE_EXPENSE_SYSTEM_FIELDS:
existing_attribute = ExpenseAttribute.objects.filter(
attribute_type=fyle_attribute, workspace_id=self.workspace_id).values_list('detail', flat=True).first()

custom_field_id = None

if existing_attribute is not None:
custom_field_id = existing_attribute['custom_field_id']

fyle_attribute = fyle_attribute.replace('_', ' ').title()
placeholder = self.construct_custom_field_placeholder(source_placeholder, fyle_attribute, existing_attribute)

expense_custom_field_payload = {
'field_name': fyle_attribute,
'type': 'SELECT',
'is_enabled': True,
'is_mandatory': False,
'placeholder': placeholder,
'options': fyle_expense_custom_field_options,
'code': None
}

if custom_field_id:
expense_field = platform.expense_custom_fields.get_by_id(custom_field_id)
expense_custom_field_payload['id'] = custom_field_id
expense_custom_field_payload['is_mandatory'] = expense_field['is_mandatory']

return expense_custom_field_payload

def construct_payload_and_import_to_fyle(
self,
platform: PlatformConnector,
import_log: ImportLog,
source_placeholder: str = None
):
"""
Construct Payload and Import to fyle in Batches
"""
filters = self.construct_attributes_filter(self.destination_field)

destination_attributes_count = DestinationAttribute.objects.filter(**filters).count()

# If there are no destination attributes, mark the import as complete
if destination_attributes_count == 0:
import_log.status = 'COMPLETE'
import_log.last_successful_run_at = datetime.now()
import_log.error_log = []
import_log.total_batches_count = 0
import_log.processed_batches_count = 0
import_log.save()
return
else:
import_log.total_batches_count = math.ceil(destination_attributes_count/200)
import_log.save()

destination_attributes_generator = self.get_destination_attributes_generator(destination_attributes_count, filters)
platform_class = self.get_platform_class(platform)

for paginated_destination_attributes, is_last_batch in destination_attributes_generator:
fyle_payload = self.construct_fyle_expense_custom_field_payload(
paginated_destination_attributes,
platform,
source_placeholder
)

self.post_to_fyle_and_sync(
fyle_payload=fyle_payload,
resource_class=platform_class,
is_last_batch=is_last_batch,
import_log=import_log
)

@handle_import_exceptions
def import_destination_attribute_to_fyle(self, import_log: ImportLog):
"""
Import destiantion_attributes field to Fyle and Auto Create Mappings
:param import_log: ImportLog object
"""

fyle_credentials = FyleCredential.objects.get(workspace_id=self.workspace_id)
platform = PlatformConnector(fyle_credentials=fyle_credentials)

self.sync_destination_attributes(self.destination_field)

self.construct_payload_and_import_to_fyle(
platform=platform,
import_log=import_log
)

self.sync_expense_attributes(platform)

self.create_mappings()
10 changes: 10 additions & 0 deletions apps/mappings/imports/queues.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ def chain_import_fields_to_fyle(workspace_id):
:param workspace_id: Workspace Id
"""
mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True)
custom_field_mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, is_custom=True, import_to_fyle=True)
configuration = Configuration.objects.get(workspace_id=workspace_id)
chain = Chain()

Expand All @@ -34,5 +35,14 @@ def chain_import_fields_to_fyle(workspace_id):
mapping_setting.source_field
)

for custom_fields_mapping_setting in custom_field_mapping_settings:
chain.append(
'apps.mappings.imports.tasks.trigger_import_via_schedule',
workspace_id,
custom_fields_mapping_setting.destination_field,
custom_fields_mapping_setting.source_field,
True
)

if chain.length() > 0:
chain.run()
12 changes: 9 additions & 3 deletions apps/mappings/imports/schedules.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import datetime, timedelta
from django_q.models import Schedule
from apps.workspaces.models import Configuration
from fyle_accounting_mappings.models import MappingSetting
Expand Down Expand Up @@ -34,8 +34,14 @@ def schedule_or_delete_fyle_import_tasks(configuration: Configuration, mapping_s
source_field__in=['CATEGORY', 'PROJECT', 'COST_CENTER']
).count()

# If the import_fields_count is 0, delete the schedule
if import_fields_count == 0:
custom_field_import_fields_count = MappingSetting.objects.filter(
import_to_fyle=True,
workspace_id=configuration.workspace_id,
is_custom=True
).count()

# If the import fields count is 0, delete the schedule
if import_fields_count == 0 and custom_field_import_fields_count == 0:
Schedule.objects.filter(
func='apps.mappings.imports.queues.chain_import_fields_to_fyle',
args='{}'.format(configuration.workspace_id)
Expand Down
13 changes: 9 additions & 4 deletions apps/mappings/imports/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@
from apps.mappings.imports.modules.projects import Project
from apps.mappings.imports.modules.categories import Category
from apps.mappings.imports.modules.cost_centers import CostCenter
from apps.mappings.imports.modules.expense_custom_fields import ExpenseCustomField

SOURCE_FIELD_CLASS_MAP = {
'PROJECT': Project,
'CATEGORY': Category,
'COST_CENTER': CostCenter,
}

def trigger_import_via_schedule(workspace_id: int, destination_field: str, source_field: str):
def trigger_import_via_schedule(workspace_id: int, destination_field: str, source_field: str, is_custom: bool = False):
"""
Trigger import via schedule
:param workspace_id: Workspace id
Expand All @@ -19,6 +20,10 @@ def trigger_import_via_schedule(workspace_id: int, destination_field: str, sourc
import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type=source_field).first()
sync_after = import_log.last_successful_run_at if import_log else None

module_class = SOURCE_FIELD_CLASS_MAP[source_field]
item = module_class(workspace_id, destination_field, sync_after)
item.trigger_import()
if is_custom:
item = ExpenseCustomField(workspace_id, source_field, destination_field, sync_after)
item.trigger_import()
else:
module_class = SOURCE_FIELD_CLASS_MAP[source_field]
item = module_class(workspace_id, destination_field, sync_after)
item.trigger_import()
Loading

0 comments on commit cabff53

Please sign in to comment.