From 35dec6475e361c3461a9875f548dc9a7c9b8e041 Mon Sep 17 00:00:00 2001 From: Ashutosh619-sudo Date: Mon, 26 Aug 2024 20:14:49 +0530 Subject: [PATCH 1/7] Max retry export with test cases --- apps/xero/queue.py | 44 +++++++++++++++-- tests/test_xero/test_tasks.py | 92 +++++++++++++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 7 deletions(-) diff --git a/apps/xero/queue.py b/apps/xero/queue.py index 0e249687..ac9ecd64 100644 --- a/apps/xero/queue.py +++ b/apps/xero/queue.py @@ -1,5 +1,6 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import List +import logging from django.db.models import Q from django_q.models import Schedule @@ -9,11 +10,24 @@ from apps.fyle.models import Expense, ExpenseGroup from apps.mappings.models import GeneralMapping from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum -from apps.tasks.models import TaskLog +from apps.tasks.models import TaskLog, Error from apps.workspaces.models import FyleCredential, XeroCredentials from apps.xero.utils import XeroConnector +logger = logging.getLogger(__name__) + +def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error): + """ + Validate failing export + :param is_auto_export: Is auto export + :param interval_hours: Interval hours + :param error: Error + """ + # If auto export is enabled and interval hours is set and error repetition count is greater than 100, export only once a day + return is_auto_export and interval_hours and error and error.repetition_count > 100 and datetime.now().replace(tzinfo=timezone.utc) - error.updated_at <= timedelta(hours=24) + + def schedule_payment_creation(sync_fyle_to_xero_payments, workspace_id): general_mappings: GeneralMapping = GeneralMapping.objects.filter( workspace_id=workspace_id @@ -106,7 +120,7 @@ def __create_chain_and_run(fyle_credentials: FyleCredential, xero_connection, in chain.run() -def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str) -> list: +def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int) -> list: """ Schedule bills creation :param expense_group_ids: List of expense group ids @@ -123,16 +137,25 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_ exported_at__isnull=True, ).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() + chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue + task_log, _ = TaskLog.objects.get_or_create( workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={"status": TaskLogStatusEnum.ENQUEUED, "type": TaskLogTypeEnum.CREATING_BILL}, ) if task_log.status not in [TaskLogStatusEnum.IN_PROGRESS, TaskLogStatusEnum.ENQUEUED]: + task_log.type = TaskLogTypeEnum.CREATING_BILL task_log.status = TaskLogStatusEnum.ENQUEUED task_log.save() @@ -173,7 +196,7 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_ def schedule_bank_transaction_creation( - workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str + workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int ) -> list: """ Schedule bank transaction creation @@ -191,19 +214,32 @@ def schedule_bank_transaction_creation( exported_at__isnull=True, ).all() + print(len(expense_groups)) + + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() + chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue + task_log, _ = TaskLog.objects.get_or_create( workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={"status": TaskLogStatusEnum.ENQUEUED, "type": TaskLogTypeEnum.CREATING_BANK_TRANSACTION}, ) + print('task_log', task_log.__dict__) if task_log.status not in [TaskLogStatusEnum.IN_PROGRESS, TaskLogStatusEnum.ENQUEUED]: + task_log.type = TaskLogTypeEnum.CREATING_BANK_TRANSACTION task_log.status = TaskLogStatusEnum.ENQUEUED task_log.save() + print('task_log', task_log.__dict__) last_export = False if expense_groups.count() == index + 1: last_export = True diff --git a/tests/test_xero/test_tasks.py b/tests/test_xero/test_tasks.py index 342c2133..a8536f70 100644 --- a/tests/test_xero/test_tasks.py +++ b/tests/test_xero/test_tasks.py @@ -9,7 +9,7 @@ from apps.fyle.models import Expense, ExpenseGroup, Reimbursement from apps.mappings.models import GeneralMapping, TenantMapping -from apps.tasks.models import TaskLog +from apps.tasks.models import Error, TaskLog from apps.workspaces.models import LastExportDetail, WorkspaceGeneralSettings, XeroCredentials from apps.xero.exceptions import update_last_export_details from apps.xero.models import BankTransaction, BankTransactionLineItem, Bill, BillLineItem @@ -416,7 +416,7 @@ def test_schedule_bills_creation(db): task_log.save() schedule_bills_creation( - workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=False, fund_source="PERSONAL" + workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=False, fund_source="PERSONAL", interval_hours=0 ) @@ -514,7 +514,7 @@ def test_schedule_bank_transaction_creation(db): task_log.save() schedule_bank_transaction_creation( - workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=False, fund_source="CCC" + workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=False, fund_source="CCC", interval_hours=0 ) @@ -1019,3 +1019,89 @@ def test__validate_expense_group(mocker, db): __validate_expense_group(expense_group) except Exception: logger.info("Mappings are missing") + + +def test_skipping_schedule_bills_creation(db): + workspace_id = 1 + + expense_group = ExpenseGroup.objects.get(id=4) + expense_group.exported_at = None + expense_group.save() + + bill = Bill.objects.get(expense_group_id=4) + BillLineItem.objects.filter(bill=bill).delete() + TaskLog.objects.filter(bill=bill).update(bill=None) + bill.delete() + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = "READY" + task_log.save() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group_id=expense_group.id, + repetition_count=106 + ) + + schedule_bills_creation( + workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=True, fund_source="CCC", interval_hours=1 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_bills_creation( + workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=True, fund_source="CCC", interval_hours=1 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_BILL' + + +def test_skipping_schedule_bank_transaction_creation(db): + workspace_id = 1 + + expense_group = ExpenseGroup.objects.get(id=5) + expense_group.exported_at = None + expense_group.save() + + bank_tran = BankTransaction.objects.get(expense_group_id=5) + BankTransactionLineItem.objects.filter(bank_transaction=bank_tran).delete() + TaskLog.objects.filter(bank_transaction=bank_tran).update(bank_transaction=None) + bank_tran.delete() + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = "READY" + task_log.save() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group_id=expense_group.id, + repetition_count=106 + ) + + schedule_bank_transaction_creation( + workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=True, fund_source="CCC", interval_hours=1 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_bank_transaction_creation( + workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=True, fund_source="CCC", interval_hours=1 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_BANK_TRANSACTION' From 8d72a0757e9f33f81cead2f795f979b2a19be280 Mon Sep 17 00:00:00 2001 From: Ashutosh619-sudo Date: Mon, 26 Aug 2024 20:20:52 +0530 Subject: [PATCH 2/7] added interval hours --- apps/workspaces/actions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/workspaces/actions.py b/apps/workspaces/actions.py index f039b928..48390d7a 100644 --- a/apps/workspaces/actions.py +++ b/apps/workspaces/actions.py @@ -205,7 +205,8 @@ def export_to_xero(workspace_id, export_mode="MANUAL", expense_group_ids=[]): workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='PERSONAL' + fund_source='PERSONAL', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) if general_settings.corporate_credit_card_expenses_object: @@ -221,7 +222,8 @@ def export_to_xero(workspace_id, export_mode="MANUAL", expense_group_ids=[]): workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='CCC' + fund_source='CCC', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) if is_expenses_exported: From 2b1408f6bf94b81dc6e49fcafa47927092d8f93c Mon Sep 17 00:00:00 2001 From: Ashutosh619-sudo Date: Mon, 26 Aug 2024 22:22:04 +0530 Subject: [PATCH 3/7] fixed flake --- apps/xero/queue.py | 3 ++- tests/test_xero/test_tasks.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/xero/queue.py b/apps/xero/queue.py index ac9ecd64..4e0cf0e8 100644 --- a/apps/xero/queue.py +++ b/apps/xero/queue.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) + def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error): """ Validate failing export @@ -227,7 +228,7 @@ def schedule_bank_transaction_creation( if skip_export: logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) continue - + task_log, _ = TaskLog.objects.get_or_create( workspace_id=expense_group.workspace_id, expense_group=expense_group, diff --git a/tests/test_xero/test_tasks.py b/tests/test_xero/test_tasks.py index a8536f70..367e84e9 100644 --- a/tests/test_xero/test_tasks.py +++ b/tests/test_xero/test_tasks.py @@ -1053,7 +1053,7 @@ def test_skipping_schedule_bills_creation(db): task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() assert task_log.type == 'FETCHING_EXPENSES' - + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) schedule_bills_creation( @@ -1096,7 +1096,7 @@ def test_skipping_schedule_bank_transaction_creation(db): task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() assert task_log.type == 'FETCHING_EXPENSES' - + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) schedule_bank_transaction_creation( From add257338aed2cdad62c5ef468bcd5ee1803cd53 Mon Sep 17 00:00:00 2001 From: Ashutosh619-sudo Date: Wed, 28 Aug 2024 16:20:59 +0530 Subject: [PATCH 4/7] change log level --- apps/xero/queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/xero/queue.py b/apps/xero/queue.py index 4e0cf0e8..4bbbbe17 100644 --- a/apps/xero/queue.py +++ b/apps/xero/queue.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) - +logger.level = logging.INFO def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error): """ From 99ec26816a7dccf2b73ef6578e6d851da2d367a0 Mon Sep 17 00:00:00 2001 From: Ashutosh619-sudo Date: Wed, 28 Aug 2024 17:10:27 +0530 Subject: [PATCH 5/7] cahnge log level --- apps/workspaces/actions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/workspaces/actions.py b/apps/workspaces/actions.py index 48390d7a..c70a19b4 100644 --- a/apps/workspaces/actions.py +++ b/apps/workspaces/actions.py @@ -27,6 +27,7 @@ from apps.xero.utils import XeroConnector logger = logging.getLogger(__name__) +logger.level = logging.INFO def post_workspace(access_token, request): From e23d6d47038031a2293db0e5f77c1d340fd00c05 Mon Sep 17 00:00:00 2001 From: Ashutosh619-sudo Date: Wed, 28 Aug 2024 17:16:59 +0530 Subject: [PATCH 6/7] flake resolved --- apps/xero/queue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/xero/queue.py b/apps/xero/queue.py index 4bbbbe17..b012aa3a 100644 --- a/apps/xero/queue.py +++ b/apps/xero/queue.py @@ -18,6 +18,7 @@ logger = logging.getLogger(__name__) logger.level = logging.INFO + def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error): """ Validate failing export From cc2493f0b1e7b888f4793f9a642f78909484d89a Mon Sep 17 00:00:00 2001 From: Ashutosh619-sudo Date: Thu, 29 Aug 2024 14:48:07 +0530 Subject: [PATCH 7/7] remove loggers --- apps/xero/queue.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/xero/queue.py b/apps/xero/queue.py index b012aa3a..5dbc0d59 100644 --- a/apps/xero/queue.py +++ b/apps/xero/queue.py @@ -216,8 +216,6 @@ def schedule_bank_transaction_creation( exported_at__isnull=True, ).all() - print(len(expense_groups)) - errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() chain_tasks = [] @@ -235,13 +233,11 @@ def schedule_bank_transaction_creation( expense_group=expense_group, defaults={"status": TaskLogStatusEnum.ENQUEUED, "type": TaskLogTypeEnum.CREATING_BANK_TRANSACTION}, ) - print('task_log', task_log.__dict__) if task_log.status not in [TaskLogStatusEnum.IN_PROGRESS, TaskLogStatusEnum.ENQUEUED]: task_log.type = TaskLogTypeEnum.CREATING_BANK_TRANSACTION task_log.status = TaskLogStatusEnum.ENQUEUED task_log.save() - print('task_log', task_log.__dict__) last_export = False if expense_groups.count() == index + 1: last_export = True