From 6221238fcc7648390eb28499708b560e386ea01c Mon Sep 17 00:00:00 2001 From: Viswas Date: Mon, 21 Oct 2024 16:34:57 +0530 Subject: [PATCH 1/7] fix: travelperk not loading all invoice profiles integrate with the travelperk pagination API to fetch invoice profiles from all pages --- apps/travelperk/connector.py | 9 ++- connectors/travelperk/apis/api_base.py | 71 +++++++------------ .../travelperk/apis/invoice_profiles.py | 25 ++++++- 3 files changed, 56 insertions(+), 49 deletions(-) diff --git a/apps/travelperk/connector.py b/apps/travelperk/connector.py index 4d7ebff2..5d199865 100644 --- a/apps/travelperk/connector.py +++ b/apps/travelperk/connector.py @@ -65,8 +65,11 @@ def sync_invoice_profile(self): :return: Dict """ - response = self.connection.invoice_profiles.get_all() - for invoice_profile in response: + profiles_generator = self.connection.invoice_profiles.get_all_generator() + profiles = [] + + for invoice_profile in profiles_generator: + profiles.append(invoice_profile) country_name = invoice_profile['billing_information']['country_name'] if 'country_name' in invoice_profile['billing_information'] else None currency = invoice_profile['currency'] if 'currency' in invoice_profile else None TravelperkProfileMapping.objects.update_or_create( @@ -79,4 +82,4 @@ def sync_invoice_profile(self): } ) - return response + return profiles diff --git a/connectors/travelperk/apis/api_base.py b/connectors/travelperk/apis/api_base.py index 814b9856..ffbb2308 100644 --- a/connectors/travelperk/apis/api_base.py +++ b/connectors/travelperk/apis/api_base.py @@ -28,7 +28,27 @@ def set_server_url(self, server_url): """ self.__server_url = server_url - def _get_request(self, object_type: str, api_url: str) -> List[Dict] or Dict: + def _get_error(self, status_code: int, response_text: str): + """Get the error object from the response. + + Parameters: + status_code (int): The status code of the response. + response_text (str): The response text. + + Returns: + The error object. + """ + error_map = { + 400: BadRequestError('Something wrong with the request body', response_text), + 401: UnauthorizedClientError('Wrong client secret or/and refresh token', response_text), + 404: NotFoundError('Client ID doesn\'t exist', response_text), + 500: InternalServerError('Internal server error', response_text), + 409: BadRequestError('The webhook already exists', response_text) + } + + return error_map.get(status_code, TravelperkError('Error: {0}'.format(status_code), response_text)) + + def _get_request(self, object_type: str, api_url: str, params: dict = {}) -> List[Dict] or Dict: """Create a HTTP GET request. Parameters: @@ -46,27 +66,15 @@ def _get_request(self, object_type: str, api_url: str) -> List[Dict] or Dict: response = requests.get( '{0}{1}'.format(self.__server_url, api_url), - headers=api_headers + headers=api_headers, + params=params ) if response.status_code == 200: result = json.loads(response.text) return result[object_type] - - elif response.status_code == 400: - raise BadRequestError('Something wrong with the request body', response.text) - - elif response.status_code == 401: - raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text) - - elif response.status_code == 404: - raise NotFoundError('Client ID doesn\'t exist', response.text) - - elif response.status_code == 500: - raise InternalServerError('Internal server error', response.text) - else: - raise TravelperkError('Error: {0}'.format(response.status_code), response.text) + raise self._get_error(response.status_code, response.text) def _post_request(self, api_url: str, data: Dict) -> Dict: """Create a HTTP POST request. @@ -95,23 +103,8 @@ def _post_request(self, api_url: str, data: Dict) -> Dict: result = json.loads(response.text) return result - elif response.status_code == 400: - raise BadRequestError('Something wrong with the request body', response.text) - - elif response.status_code == 401: - raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text) - - elif response.status_code == 404: - raise NotFoundError('Client ID doesn\'t exist', response.text) - - elif response.status_code == 500: - raise InternalServerError('Internal server error', response.text) - - elif response.status_code == 409: - raise BadRequestError('The webhook already exists', response.text) - else: - raise TravelperkError('Error: {0}'.format(response.status_code), response.text) + raise self._get_error(response.status_code, response.text) def _delete_request(self, api_url: str) -> Dict: """Create a HTTP DELETE request. @@ -137,17 +130,5 @@ def _delete_request(self, api_url: str) -> Dict: if response.status_code == 200: return response.text - elif response.status_code == 400: - raise BadRequestError('Something wrong with the request body', response.text) - - elif response.status_code == 401: - raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text) - - elif response.status_code == 404: - raise NotFoundError('Client ID doesn\'t exist', response.text) - - elif response.status_code == 500: - raise InternalServerError('Internal server error', response.text) - else: - raise TravelperkError('Error: {0}'.format(response.status_code), response.text) \ No newline at end of file + raise self._get_error(response.status_code, response.text) diff --git a/connectors/travelperk/apis/invoice_profiles.py b/connectors/travelperk/apis/invoice_profiles.py index 573d73e5..b745ecd9 100644 --- a/connectors/travelperk/apis/invoice_profiles.py +++ b/connectors/travelperk/apis/invoice_profiles.py @@ -15,4 +15,27 @@ def get_all(self): Returns: List with dicts in Invoice Profile schema. """ - return self._get_request('profiles', InvoiceProfiles.GET_INVOICE_PROFILES) + return [*self.get_all_generator()] + + def get_all_generator(self): + """ + Creates a generator that contains all profiles across all pages + + Parameters: + object_type (str): The type of object to get + api_url (str): The url for the wanted API + params (dict): The parameters for the request + + Returns: + Generator with all objects of type `object_type` + """ + + limit = 50 + params = {'limit': limit} + total = self._get_request('total', InvoiceProfiles.GET_INVOICE_PROFILES, params=params) + + for offset in range(0, total, limit): + params['offset'] = offset + profiles = self._get_request('profiles', InvoiceProfiles.GET_INVOICE_PROFILES, params=params) + for profile in profiles: + yield profile From 468c11963c4f15645e6bf7b399f03d1476834223 Mon Sep 17 00:00:00 2001 From: Viswas Date: Mon, 21 Oct 2024 19:53:12 +0530 Subject: [PATCH 2/7] refactor: move `get_all_generator` to `ApiBase` and rename variable --- connectors/travelperk/apis/api_base.py | 26 +++++++++++++++++++ .../travelperk/apis/invoice_profiles.py | 25 ------------------ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/connectors/travelperk/apis/api_base.py b/connectors/travelperk/apis/api_base.py index ffbb2308..c1fbbb24 100644 --- a/connectors/travelperk/apis/api_base.py +++ b/connectors/travelperk/apis/api_base.py @@ -8,6 +8,8 @@ class ApiBase: """The base class for all API classes.""" + GET_INVOICE_PROFILES = '/profiles' + def __init__(self): self.__access_token = None self.__server_url = None @@ -76,6 +78,30 @@ def _get_request(self, object_type: str, api_url: str, params: dict = {}) -> Lis else: raise self._get_error(response.status_code, response.text) + def get_all_generator(self): + """ + Creates a generator that contains all profiles across all pages + + Parameters: + object_type (str): The type of object to get + api_url (str): The url for the wanted API + params (dict): The parameters for the request + + Returns: + Generator with all objects of type `object_type` + """ + + limit = 50 + params = {'limit': limit} + total_profiles = self._get_request('total', self.GET_INVOICE_PROFILES, params=params) + + for offset in range(0, total_profiles, limit): + params['offset'] = offset + profiles = self._get_request('profiles', self.GET_INVOICE_PROFILES, params=params) + for profile in profiles: + print('[x]', profile['name']) + yield profile + def _post_request(self, api_url: str, data: Dict) -> Dict: """Create a HTTP POST request. diff --git a/connectors/travelperk/apis/invoice_profiles.py b/connectors/travelperk/apis/invoice_profiles.py index b745ecd9..31e750ce 100644 --- a/connectors/travelperk/apis/invoice_profiles.py +++ b/connectors/travelperk/apis/invoice_profiles.py @@ -7,8 +7,6 @@ class InvoiceProfiles(ApiBase): """Class for Invoice Profiles APIs.""" - GET_INVOICE_PROFILES = '/profiles' - def get_all(self): """Get a list of the existing Invoice Profiles in the Organization. @@ -16,26 +14,3 @@ def get_all(self): List with dicts in Invoice Profile schema. """ return [*self.get_all_generator()] - - def get_all_generator(self): - """ - Creates a generator that contains all profiles across all pages - - Parameters: - object_type (str): The type of object to get - api_url (str): The url for the wanted API - params (dict): The parameters for the request - - Returns: - Generator with all objects of type `object_type` - """ - - limit = 50 - params = {'limit': limit} - total = self._get_request('total', InvoiceProfiles.GET_INVOICE_PROFILES, params=params) - - for offset in range(0, total, limit): - params['offset'] = offset - profiles = self._get_request('profiles', InvoiceProfiles.GET_INVOICE_PROFILES, params=params) - for profile in profiles: - yield profile From 26faa2a80e3a595a781fe59fe5e86d018cba9787 Mon Sep 17 00:00:00 2001 From: Viswas Date: Tue, 22 Oct 2024 16:52:44 +0530 Subject: [PATCH 3/7] fix: generalise the `get_all_generator` method for all onject types and create an implementation in `InvoiceProfiles` --- connectors/travelperk/apis/api_base.py | 11 ++++------- connectors/travelperk/apis/invoice_profiles.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/connectors/travelperk/apis/api_base.py b/connectors/travelperk/apis/api_base.py index c1fbbb24..ba7879a9 100644 --- a/connectors/travelperk/apis/api_base.py +++ b/connectors/travelperk/apis/api_base.py @@ -8,8 +8,6 @@ class ApiBase: """The base class for all API classes.""" - GET_INVOICE_PROFILES = '/profiles' - def __init__(self): self.__access_token = None self.__server_url = None @@ -78,9 +76,9 @@ def _get_request(self, object_type: str, api_url: str, params: dict = {}) -> Lis else: raise self._get_error(response.status_code, response.text) - def get_all_generator(self): + def _get_all_generator(self, object_type: str, api_url: str): """ - Creates a generator that contains all profiles across all pages + Creates a generator that contains all records of `object_type` across all pages Parameters: object_type (str): The type of object to get @@ -93,13 +91,12 @@ def get_all_generator(self): limit = 50 params = {'limit': limit} - total_profiles = self._get_request('total', self.GET_INVOICE_PROFILES, params=params) + total_profiles = self._get_request('total', api_url, params=params) for offset in range(0, total_profiles, limit): params['offset'] = offset - profiles = self._get_request('profiles', self.GET_INVOICE_PROFILES, params=params) + profiles = self._get_request(object_type, api_url, params=params) for profile in profiles: - print('[x]', profile['name']) yield profile def _post_request(self, api_url: str, data: Dict) -> Dict: diff --git a/connectors/travelperk/apis/invoice_profiles.py b/connectors/travelperk/apis/invoice_profiles.py index 31e750ce..e042415f 100644 --- a/connectors/travelperk/apis/invoice_profiles.py +++ b/connectors/travelperk/apis/invoice_profiles.py @@ -7,10 +7,20 @@ class InvoiceProfiles(ApiBase): """Class for Invoice Profiles APIs.""" + GET_INVOICE_PROFILES = '/profiles' + def get_all(self): """Get a list of the existing Invoice Profiles in the Organization. Returns: List with dicts in Invoice Profile schema. """ - return [*self.get_all_generator()] + return [*self._get_all_generator('profiles', self.GET_INVOICE_PROFILES)] + + def get_all_generator(self): + """Create a generator with all the existing Invoice Profiles in the Organization. + + Returns: + Generator with dicts in Invoice Profile schema. + """ + return self._get_all_generator('profiles', self.GET_INVOICE_PROFILES) \ No newline at end of file From 513a649f235756f4c212dc81c1b516149d8c352b Mon Sep 17 00:00:00 2001 From: Viswas Date: Tue, 22 Oct 2024 16:55:06 +0530 Subject: [PATCH 4/7] refactor: clarity --- connectors/travelperk/apis/invoice_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connectors/travelperk/apis/invoice_profiles.py b/connectors/travelperk/apis/invoice_profiles.py index e042415f..a28d37c2 100644 --- a/connectors/travelperk/apis/invoice_profiles.py +++ b/connectors/travelperk/apis/invoice_profiles.py @@ -15,7 +15,7 @@ def get_all(self): Returns: List with dicts in Invoice Profile schema. """ - return [*self._get_all_generator('profiles', self.GET_INVOICE_PROFILES)] + return [*self.get_all_generator()] def get_all_generator(self): """Create a generator with all the existing Invoice Profiles in the Organization. From ba9d32cb08c9a95dba83e38d3cbcd9d8dd1bf6a4 Mon Sep 17 00:00:00 2001 From: Viswas Date: Thu, 24 Oct 2024 12:47:20 +0530 Subject: [PATCH 5/7] fix: apply travelperk sync fix to all existing accounts --- scripts/python/sync_all_travelperk_accounts.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 scripts/python/sync_all_travelperk_accounts.py diff --git a/scripts/python/sync_all_travelperk_accounts.py b/scripts/python/sync_all_travelperk_accounts.py new file mode 100644 index 00000000..1b4d626a --- /dev/null +++ b/scripts/python/sync_all_travelperk_accounts.py @@ -0,0 +1,8 @@ +from apps.travelperk.models import TravelperkCredential +from apps.travelperk.serializers import SyncPaymentProfileSerializer + + +all_creds = TravelperkCredential.objects.all() + +for creds in all_creds: + SyncPaymentProfileSerializer().sync_payment_profiles(creds.org_id) From 600869d73aa0c87c409083037a7c00372aeaecc3 Mon Sep 17 00:00:00 2001 From: Viswas Date: Mon, 28 Oct 2024 17:32:00 +0530 Subject: [PATCH 6/7] test: add request and response logs to the travelperk api --- connectors/travelperk/apis/api_base.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/connectors/travelperk/apis/api_base.py b/connectors/travelperk/apis/api_base.py index ba7879a9..c110a9de 100644 --- a/connectors/travelperk/apis/api_base.py +++ b/connectors/travelperk/apis/api_base.py @@ -1,9 +1,11 @@ +import logging from typing import List, Dict import requests import json from connectors.travelperk.exceptions import * +logger = logging.getLogger(__name__) class ApiBase: """The base class for all API classes.""" @@ -28,7 +30,7 @@ def set_server_url(self, server_url): """ self.__server_url = server_url - def _get_error(self, status_code: int, response_text: str): + def _get_error(self, status_code: int, response_text: str) -> TravelperkError: """Get the error object from the response. Parameters: @@ -116,18 +118,25 @@ def _post_request(self, api_url: str, data: Dict) -> Dict: 'Api-Version': '1' } + endpoint = '{0}{1}'.format(self.__server_url, api_url) + logger.debug(f"POST {endpoint}") + logger.debug(f"Payload for POST request: {data}") response = requests.post( - '{0}{1}'.format(self.__server_url, api_url), + endpoint, headers=api_headers, json=data ) if response.status_code == 200: result = json.loads(response.text) + logger.debug(f"POST response: {result}") return result else: - raise self._get_error(response.status_code, response.text) + error = self._get_error(response.status_code, response.text) + logger.info(f"POST request failed: {response.status_code} | {error.message}") + logger.info(f"POST response: {error.response}") + raise error def _delete_request(self, api_url: str) -> Dict: """Create a HTTP DELETE request. From dfa4e99dddc26ab3a9baa8a4c4f6b1b3704ae3af Mon Sep 17 00:00:00 2001 From: Viswas Date: Mon, 28 Oct 2024 20:00:38 +0530 Subject: [PATCH 7/7] fix: add loggers for GET and DELETE requests as well --- connectors/travelperk/apis/api_base.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/connectors/travelperk/apis/api_base.py b/connectors/travelperk/apis/api_base.py index c110a9de..7747d134 100644 --- a/connectors/travelperk/apis/api_base.py +++ b/connectors/travelperk/apis/api_base.py @@ -66,17 +66,24 @@ def _get_request(self, object_type: str, api_url: str, params: dict = {}) -> Lis 'Api-Version': '1' } + endpoint = '{0}{1}'.format(self.__server_url, api_url) + logger.debug(f"GET {endpoint}") + logger.debug(f"Params for GET request: {params}") response = requests.get( - '{0}{1}'.format(self.__server_url, api_url), + endpoint, headers=api_headers, params=params ) if response.status_code == 200: result = json.loads(response.text) + logger.debug(f"GET response: {result}") return result[object_type] else: - raise self._get_error(response.status_code, response.text) + error = self._get_error(response.status_code, response.text) + logger.info(f"GET request failed: {response.status_code} | {error.message}") + logger.info(f"GET response: {error.response}") + raise error def _get_all_generator(self, object_type: str, api_url: str): """ @@ -154,13 +161,18 @@ def _delete_request(self, api_url: str) -> Dict: 'Api-Version': '1' } + endpoint = '{0}{1}'.format(self.__server_url, api_url) + logger.debug(f"DELETE {endpoint}") response = requests.delete( - '{0}{1}'.format(self.__server_url, api_url), + endpoint, headers=api_headers ) if response.status_code == 200: + logger.debug(f"DELETE response: {response.text}") return response.text - else: - raise self._get_error(response.status_code, response.text) + error = self._get_error(response.status_code, response.text) + logger.info(f"DELETE request failed: {response.status_code} | {error.message}") + logger.info(f"DELETE response: {error.response}") + raise error