fix: travelperk not loading all invoice profiles (#219)

* fix: travelperk not loading all invoice profiles

integrate with the travelperk pagination API to fetch invoice profiles from all pages

* refactor: move `get_all_generator` to `ApiBase` and rename variable

* fix: generalise the `get_all_generator` method for all onject types and create an implementation in `InvoiceProfiles`

* refactor: clarity

apps/travelperk/connector.py

@@ -65,8 +65,11 @@ def sync_invoice_profile(self):
         :return: Dict
         """
 
-        response = self.connection.invoice_profiles.get_all()
-        for invoice_profile in response:
+        profiles_generator = self.connection.invoice_profiles.get_all_generator()
+        profiles = []
+
+        for invoice_profile in profiles_generator:
+            profiles.append(invoice_profile)
             country_name = invoice_profile['billing_information']['country_name'] if 'country_name' in invoice_profile['billing_information'] else None
             currency = invoice_profile['currency'] if 'currency' in invoice_profile else None
             TravelperkProfileMapping.objects.update_or_create(
@@ -79,4 +82,4 @@ def sync_invoice_profile(self):
                 }
             )
 
-        return response
+        return profiles

connectors/travelperk/apis/api_base.py

@@ -28,7 +28,27 @@ def set_server_url(self, server_url):
         """
         self.__server_url = server_url
 
-    def _get_request(self, object_type: str, api_url: str) -> List[Dict] or Dict:
+    def _get_error(self, status_code: int, response_text: str):
+        """Get the error object from the response.
+
+        Parameters:
+            status_code (int): The status code of the response.
+            response_text (str): The response text.
+
+        Returns:
+            The error object.
+        """
+        error_map = {
+            400: BadRequestError('Something wrong with the request body', response_text),
+            401: UnauthorizedClientError('Wrong client secret or/and refresh token', response_text),
+            404: NotFoundError('Client ID doesn\'t exist', response_text),
+            500: InternalServerError('Internal server error', response_text),
+            409: BadRequestError('The webhook already exists', response_text)
+        }
+
+        return error_map.get(status_code, TravelperkError('Error: {0}'.format(status_code), response_text))
+
+    def _get_request(self, object_type: str, api_url: str, params: dict = {}) -> List[Dict] or Dict:
         """Create a HTTP GET request.
 
         Parameters:
@@ -46,27 +66,38 @@ def _get_request(self, object_type: str, api_url: str) -> List[Dict] or Dict:
 
         response = requests.get(
             '{0}{1}'.format(self.__server_url, api_url),
-            headers=api_headers
+            headers=api_headers,
+            params=params
         )
 
         if response.status_code == 200:
             result = json.loads(response.text)
             return result[object_type]
+        else:
+            raise self._get_error(response.status_code, response.text)
 
-        elif response.status_code == 400:
-            raise BadRequestError('Something wrong with the request body', response.text)
-
-        elif response.status_code == 401:
-            raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text)
+    def _get_all_generator(self, object_type: str, api_url: str):
+        """
+        Creates a generator that contains all records of `object_type` across all pages
+        
+        Parameters:
+            object_type (str): The type of object to get
+            api_url (str): The url for the wanted API
+            params (dict): The parameters for the request
 
-        elif response.status_code == 404:
-            raise NotFoundError('Client ID doesn\'t exist', response.text)
+        Returns:
+            Generator with all objects of type `object_type`
+        """
 
-        elif response.status_code == 500:
-            raise InternalServerError('Internal server error', response.text)
+        limit = 50
+        params = {'limit': limit}
+        total_profiles = self._get_request('total', api_url, params=params)
 
-        else:
-            raise TravelperkError('Error: {0}'.format(response.status_code), response.text)
+        for offset in range(0, total_profiles, limit):
+            params['offset'] = offset
+            profiles = self._get_request(object_type, api_url, params=params)
+            for profile in profiles:
+                yield profile
 
     def _post_request(self, api_url: str, data: Dict) -> Dict:
         """Create a HTTP POST request.
@@ -95,23 +126,8 @@ def _post_request(self, api_url: str, data: Dict) -> Dict:
             result = json.loads(response.text)
             return result
 
-        elif response.status_code == 400:
-            raise BadRequestError('Something wrong with the request body', response.text)
-
-        elif response.status_code == 401:
-            raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text)
-
-        elif response.status_code == 404:
-            raise NotFoundError('Client ID doesn\'t exist', response.text)
-
-        elif response.status_code == 500:
-            raise InternalServerError('Internal server error', response.text)
-
-        elif response.status_code == 409:
-            raise BadRequestError('The webhook already exists', response.text)
-
         else:
-            raise TravelperkError('Error: {0}'.format(response.status_code), response.text)
+            raise self._get_error(response.status_code, response.text)
 
     def _delete_request(self, api_url: str) -> Dict:
         """Create a HTTP DELETE request.
@@ -137,17 +153,5 @@ def _delete_request(self, api_url: str) -> Dict:
         if response.status_code == 200:
             return response.text
 
-        elif response.status_code == 400:
-            raise BadRequestError('Something wrong with the request body', response.text)
-
-        elif response.status_code == 401:
-            raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text)
-
-        elif response.status_code == 404:
-            raise NotFoundError('Client ID doesn\'t exist', response.text)
-
-        elif response.status_code == 500:
-            raise InternalServerError('Internal server error', response.text)
-
         else:
-            raise TravelperkError('Error: {0}'.format(response.status_code), response.text)
+            raise self._get_error(response.status_code, response.text)

connectors/travelperk/apis/invoice_profiles.py

@@ -15,4 +15,12 @@ def get_all(self):
         Returns:
             List with dicts in Invoice Profile schema.
         """
-        return self._get_request('profiles', InvoiceProfiles.GET_INVOICE_PROFILES)
+        return [*self.get_all_generator()]
+
+    def get_all_generator(self):
+        """Create a generator with all the existing Invoice Profiles in the Organization.
+
+        Returns:
+            Generator with dicts in Invoice Profile schema.
+        """
+        return self._get_all_generator('profiles', self.GET_INVOICE_PROFILES)