Merge pull request #704 from MetaCell/feature/700

Fix and improve api tests

application-templates/django-app/api/test_st.py

@@ -0,0 +1,27 @@
+import os
+from pprint import pprint
+import schemathesis as st
+from schemathesis.checks import response_schema_conformance, not_a_server_error
+
+from cloudharness_test import apitest_init # include to perform default authorization
+
+app_url = os.environ.get("APP_URL", "http://samples.ch.local/api")
+
+try:
+    schema = st.from_uri(app_url + "/openapi.json")
+except:
+    # support alternative schema location
+    schema = st.from_uri(app_url.replace("/api", "") + "/openapi.json")
+
+
+@schema.parametrize(endpoint="/ping")
+def test_ping(case):
+    response = case.call()
+    pprint(response.__dict__)
+    assert response.status_code == 200, "this api errors on purpose"
+
+def test_state_machine():
+    schema.as_state_machine().run()
+# APIWorkflow = schema.as_state_machine()
+# APIWorkflow.run()
+# TestAPI = APIWorkflow.TestCase

deployment-configuration/value-template.yaml

@@ -22,6 +22,8 @@ harness:
         - administrator
     - uri: /api/openapi.json
       white-listed: true
+    - uri: /openapi.json
+      white-listed: true
   # -- Defines reference deployment parameters. Values maps to k8s spec
   deployment:
     # -- When true, enables automatic deployment

docs/testing.md

@@ -108,23 +108,30 @@ The test can use environmental variables:
 Examples:
 - [Sample api test](../applications/samples/test/api/test_st.py)
 
-### Common smoke tests
-
-Once a test is created for your application, generic smoke tests are also
-executed, checking for:
 
-- Main page is reachable
-- No errors in the console
-- No error responses from network resources and fetch requests (code < 400)
-
-The smoke tests is defined [in this file](../test/test-e2e/__tests__/common.spec.ts).
 
 
 
 ## End to end (E2E) tests
 
 End to end tests run in a headless browser ([Puppeteer](https://github.com/puppeteer/puppeteer)) against the full deployment on Kubernetes.
 
+Custom configuration:
+
+```yaml
+harness:
+  ...
+  test:
+    e2e:
+      # -- enable/disable e2e tests
+      enabled: true
+      # -- ignore errors on console by default
+      ignoreConsoleErrors: false
+      # -- ignore fetched resources errors by default
+      ignoreRequestErrors: false
+      # -- enable common smoke tests
+      smoketest: true
+```
 
 ### Write tests with Jest and Puppeteer
 
@@ -159,7 +166,7 @@ executed, checking for:
 - No errors in the console
 - No error responses from network resources and fetch requests (code < 400)
 
-The smoke tests is defined [in this file](../test/jest-puppeteer/__tests__/common.spec.ts).
+The smoke tests are defined [in this file](../test/jest-puppeteer/__tests__/common.spec.ts).
 
 
 ## Run API and E2E tests in the CI/CD pipeline
@@ -182,7 +189,7 @@ deployment.
 In order to use `harness-test` install the library with
 
 ```
-pip install -r requirements-test.txt
+pip install -e tools/cloudharness-test
 ```
 
 In order to run tests against an existing deployment based on a domain (say, my.domain), run:
@@ -198,6 +205,25 @@ If you want to run the deployment locally and then run the tests, can use skaffo
 1. Wait the deployment to settle
 1. Run `harness-test PATHS`
 
+### Tests development
+The `harness-test` client is useful while developing and tweaking the tests.
+In that case it's better to target the application under development and 
+the kind of tests we are working on.
+
+
+To target a specific application for end-to-end tests, use:
+```
+harness-test . -i [APPNAME] -e
+```
+
+To target a specific application for api tests, use:
+```
+harness-test . -i [APPNAME] -a
+```
+
+Note that the local version of the openapi.yaml file located at applications/[APPNAME]/api/openapi.yaml is used if available. That's useful to tweak examples and responses
+used by schemathesis to generate the test hypotheses.
+
 ## Create test users for your application
 
 To create test users:

tools/cloudharness-test/cloudharness_test/api.py

@@ -60,6 +60,13 @@ def run_api_tests(root_paths, helm_values: HarnessMainConfig, base_domain, inclu
 
             app_env = get_app_environment(app_config, app_domain)
 
+            schema_file = f"applications/{app_config.name}/api/openapi.yaml"
+
+            for path in root_paths:
+                # use local schema if available to simplify test development
+                if os.path.exists(os.path.join(path, schema_file)):
+                    app_env["APP_SCHEMA_FILE"] = schema_file
+
             if api_config.autotest:
                 logging.info("Running auto api tests")
                 cmd = get_schemathesis_command(api_filename, app_config, app_domain)

tools/cloudharness-test/cloudharness_test/apitest_init.py

@@ -1,44 +1,64 @@
 import os
 import logging
+import requests
 
 import schemathesis as st
 
 from cloudharness.auth import get_token
 
-if "APP_URL" in os.environ:
+if "APP_URL" or "APP_SCHEMA_FILE" in os.environ:
+    app_schema = os.environ.get("APP_SCHEMA_FILE", None)
+    app_url = os.environ.get("APP_URL", "http://samples.ch.local/api")
+    logging.info("Start schemathesis tests on %s", app_url)
+    if app_schema:
+        # Test locally with harness-test -- use local schema for convenience during test development
+        openapi_uri = app_schema
+        schema = st.from_file(openapi_uri)
+    else:
+        # remote testing: might be /api/openapi.json or /openapi.json
+        try:
+            openapi_uri = openapi_uri = app_url + "/openapi.json"
+            schema = st.from_uri(openapi_uri)
+        except st.exceptions.SchemaLoadingError as e:
+            # Use alternative configuration
+            try:
+                openapi_uri = app_url.replace("/api", "") + "/openapi.json"
+                print(requests.get(openapi_uri))
+                schema = st.from_uri(openapi_uri)
+            except st.exceptions.SchemaLoadingError as e:
+                raise Exception(
+                    f"Cannot setup api tests: {openapi_uri} not valid. Check your deployment is up and configuration") from e
 
-    app_url = os.environ["APP_URL"]
+        except Exception as e:
+            raise Exception(
+                f"Cannot setup api tests: {openapi_uri}: {e}") from e
 
-    try:
-        openapi_uri = app_url + "/openapi.json"
-        logging.info("Using openapi spec at %s", openapi_uri)
-        schema = st.from_uri(openapi_uri)
-    except Exception as e:
-        raise Exception(f"Cannot setup api tests: {openapi_uri} not reachable. Check your deployment is up and configuration") from e
+    logging.info("Using openapi spec at %s", openapi_uri)
 
     if "USERNAME" in os.environ and "PASSWORD" in os.environ:
         logging.info("Setting token from username and password")
+
         @st.auth.register()
         class TokenAuth:
             def get(self, context):
-                
+
                 username = os.environ["USERNAME"]
-                password = os.environ["PASSWORD"]  
-        
+                password = os.environ["PASSWORD"]
+
                 return get_token(username, password)
 
             def set(self, case, data, context):
                 case.headers = case.headers or {}
-                case.headers["Authorization"]  = f"Bearer {data}"
-                case.headers["Cookie"]  = f"kc-access={data}"
+                case.headers["Authorization"] = f"Bearer {data}"
+                case.headers["Cookie"] = f"kc-access={data}"
     else:
         @st.auth.register()
         class TokenAuth:
             def get(self, context):
-                
+
                 return ""
 
             def set(self, case, data, context):
                 case.headers = case.headers or {}
-                case.headers["Authorization"]  = f"Bearer {data}"
-                case.headers["Cookie"]  = f"kc-access={data}"
+                case.headers["Authorization"] = f"Bearer {data}"
+                case.headers["Cookie"] = f"kc-access={data}"