From 2582aa0bb693672e8851db791f0e6e41e59f503c Mon Sep 17 00:00:00 2001 From: Ashutosh singh <55102089+Ashutosh619-sudo@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:31:21 +0530 Subject: [PATCH] Max retry exports (#391) * Max retry export with test cases * added interval hours * fixed flake * change log level * cahnge log level * flake resolved * remove loggers --- apps/workspaces/actions.py | 7 ++- apps/xero/queue.py | 42 ++++++++++++++-- tests/test_xero/test_tasks.py | 92 +++++++++++++++++++++++++++++++++-- 3 files changed, 132 insertions(+), 9 deletions(-) diff --git a/apps/workspaces/actions.py b/apps/workspaces/actions.py index f039b928..c70a19b4 100644 --- a/apps/workspaces/actions.py +++ b/apps/workspaces/actions.py @@ -27,6 +27,7 @@ from apps.xero.utils import XeroConnector logger = logging.getLogger(__name__) +logger.level = logging.INFO def post_workspace(access_token, request): @@ -205,7 +206,8 @@ def export_to_xero(workspace_id, export_mode="MANUAL", expense_group_ids=[]): workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='PERSONAL' + fund_source='PERSONAL', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) if general_settings.corporate_credit_card_expenses_object: @@ -221,7 +223,8 @@ def export_to_xero(workspace_id, export_mode="MANUAL", expense_group_ids=[]): workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='CCC' + fund_source='CCC', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) if is_expenses_exported: diff --git a/apps/xero/queue.py b/apps/xero/queue.py index 0e249687..5dbc0d59 100644 --- a/apps/xero/queue.py +++ b/apps/xero/queue.py @@ -1,5 +1,6 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import List +import logging from django.db.models import Q from django_q.models import Schedule @@ -9,11 +10,26 @@ from apps.fyle.models import Expense, ExpenseGroup from apps.mappings.models import GeneralMapping from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum -from apps.tasks.models import TaskLog +from apps.tasks.models import TaskLog, Error from apps.workspaces.models import FyleCredential, XeroCredentials from apps.xero.utils import XeroConnector +logger = logging.getLogger(__name__) +logger.level = logging.INFO + + +def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error): + """ + Validate failing export + :param is_auto_export: Is auto export + :param interval_hours: Interval hours + :param error: Error + """ + # If auto export is enabled and interval hours is set and error repetition count is greater than 100, export only once a day + return is_auto_export and interval_hours and error and error.repetition_count > 100 and datetime.now().replace(tzinfo=timezone.utc) - error.updated_at <= timedelta(hours=24) + + def schedule_payment_creation(sync_fyle_to_xero_payments, workspace_id): general_mappings: GeneralMapping = GeneralMapping.objects.filter( workspace_id=workspace_id @@ -106,7 +122,7 @@ def __create_chain_and_run(fyle_credentials: FyleCredential, xero_connection, in chain.run() -def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str) -> list: +def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int) -> list: """ Schedule bills creation :param expense_group_ids: List of expense group ids @@ -123,16 +139,25 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_ exported_at__isnull=True, ).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() + chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue + task_log, _ = TaskLog.objects.get_or_create( workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={"status": TaskLogStatusEnum.ENQUEUED, "type": TaskLogTypeEnum.CREATING_BILL}, ) if task_log.status not in [TaskLogStatusEnum.IN_PROGRESS, TaskLogStatusEnum.ENQUEUED]: + task_log.type = TaskLogTypeEnum.CREATING_BILL task_log.status = TaskLogStatusEnum.ENQUEUED task_log.save() @@ -173,7 +198,7 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_ def schedule_bank_transaction_creation( - workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str + workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int ) -> list: """ Schedule bank transaction creation @@ -191,16 +216,25 @@ def schedule_bank_transaction_creation( exported_at__isnull=True, ).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() + chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue + task_log, _ = TaskLog.objects.get_or_create( workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={"status": TaskLogStatusEnum.ENQUEUED, "type": TaskLogTypeEnum.CREATING_BANK_TRANSACTION}, ) if task_log.status not in [TaskLogStatusEnum.IN_PROGRESS, TaskLogStatusEnum.ENQUEUED]: + task_log.type = TaskLogTypeEnum.CREATING_BANK_TRANSACTION task_log.status = TaskLogStatusEnum.ENQUEUED task_log.save() diff --git a/tests/test_xero/test_tasks.py b/tests/test_xero/test_tasks.py index 342c2133..367e84e9 100644 --- a/tests/test_xero/test_tasks.py +++ b/tests/test_xero/test_tasks.py @@ -9,7 +9,7 @@ from apps.fyle.models import Expense, ExpenseGroup, Reimbursement from apps.mappings.models import GeneralMapping, TenantMapping -from apps.tasks.models import TaskLog +from apps.tasks.models import Error, TaskLog from apps.workspaces.models import LastExportDetail, WorkspaceGeneralSettings, XeroCredentials from apps.xero.exceptions import update_last_export_details from apps.xero.models import BankTransaction, BankTransactionLineItem, Bill, BillLineItem @@ -416,7 +416,7 @@ def test_schedule_bills_creation(db): task_log.save() schedule_bills_creation( - workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=False, fund_source="PERSONAL" + workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=False, fund_source="PERSONAL", interval_hours=0 ) @@ -514,7 +514,7 @@ def test_schedule_bank_transaction_creation(db): task_log.save() schedule_bank_transaction_creation( - workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=False, fund_source="CCC" + workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=False, fund_source="CCC", interval_hours=0 ) @@ -1019,3 +1019,89 @@ def test__validate_expense_group(mocker, db): __validate_expense_group(expense_group) except Exception: logger.info("Mappings are missing") + + +def test_skipping_schedule_bills_creation(db): + workspace_id = 1 + + expense_group = ExpenseGroup.objects.get(id=4) + expense_group.exported_at = None + expense_group.save() + + bill = Bill.objects.get(expense_group_id=4) + BillLineItem.objects.filter(bill=bill).delete() + TaskLog.objects.filter(bill=bill).update(bill=None) + bill.delete() + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = "READY" + task_log.save() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group_id=expense_group.id, + repetition_count=106 + ) + + schedule_bills_creation( + workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=True, fund_source="CCC", interval_hours=1 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_bills_creation( + workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=True, fund_source="CCC", interval_hours=1 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_BILL' + + +def test_skipping_schedule_bank_transaction_creation(db): + workspace_id = 1 + + expense_group = ExpenseGroup.objects.get(id=5) + expense_group.exported_at = None + expense_group.save() + + bank_tran = BankTransaction.objects.get(expense_group_id=5) + BankTransactionLineItem.objects.filter(bank_transaction=bank_tran).delete() + TaskLog.objects.filter(bank_transaction=bank_tran).update(bank_transaction=None) + bank_tran.delete() + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = "READY" + task_log.save() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group_id=expense_group.id, + repetition_count=106 + ) + + schedule_bank_transaction_creation( + workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=True, fund_source="CCC", interval_hours=1 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_bank_transaction_creation( + workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=True, fund_source="CCC", interval_hours=1 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_BANK_TRANSACTION'