ci: update build workflows to use image caching, multi-arch, auto-tag (…

…#925)

* refactor: move scripts & josm to contrib dir

* ci: add contrib label to labeler.yml

* ci: separate frontend and backend tests during pr

* ci: remove extract-vars (use auto-tagging)

* build: update dockerfiles and compose to use COMMIT_REFs

* ci: build backend images on tag and release events

* ci: publish docs on changes to development branch

* build: set default S3_ENDPOINT for entrypoint (if not set)

* build: add healthchecks to dockerfiles

* ci: enable backend smoketest using compose --wait

* build: update odkcentral --> v2023.4.0

* build: healthcheck for minio service

* ci(pytest): update pytest workflow to use img artifacts

* build: update odkcentral img, add entrypoint + healthcheck

* ci: fix pytest img tar downloading

* ci(pytest): fix cached image loading from /tmp

* ci(pytest): simplify image caching and loading

* ci(pytest): remove image cache upload

* ci(pytest): add image cache job to end of workflow

* ci(pytest): set cache loading based on needs outputs

* ci(pytest): cache images if download was unsuccessful

* ci(pytest): fix operator !=

* build: use env vars for all image versions

* build: ODK_CENTRAL_VERSION --> ODK_CENTRAL_TAG

* ci(pytest): set image tags for caching from github vars

* ci: replace deploy workflow with reusable

* ci(pytest): image caching cache_name --> artifact_name

* ci(deploy): set .env from var and secrets context (not manual)

* ci(pytest): increase timeout for central wait-for-it 30s

* ci(pytest): fix timeout syntax

* build: add shebang to odkcentral entrypoint script

* ci: update env to .env for pytest

* ci(pytest): remove additional -alpine from postgis img

* build: redact password in odk-central entrypoint

* ci: fix to_envs single .env file parsing

* ci: pass env var directly to jq, not github var

* ci(pytest): fix by using scalar over literal syntax

* build: hardcode WORKER_COUNT=1 for odk-central start

* ci: update docs workflow to use new cahcing

* docs: update info for testing workflows via act

* ci(pytest): add caching img steps directly to workflow (no reuse)

* ci(pytest): typo image --> images for cache key

* refactor: pydantic deprecation FIeldValidationInfo --> ValidationInfo

* ci: rewrite pytest to build pr image prior to test

* ci: don't push,cache,multi_arch PR image builds

* ci: add permissions: write to pytest build + scan=false

* ci: remove permissions from pytest (handle downstream)

* ci: update conditionals for building pr images

* ci: update pytest cache-hit output

* ci: get image_tag instead of image_name from output

* ci(pytest): fix case where cache hit, skip build

* ci(pytest): set BACKEND_IMG_TAG in all if cases

* ci(pytest): replace BACKEND_IMG_TAG --> API_TAG_OVERRIDE

* ci(pytest): fix getting backend image tag from docker img ls

* ci(pytest): only run image caching steps if explicity cache=false

* ci(pytest): revert ==false to !=true negation

* ci: replace all refs to repo with gh vars

* ci(pytest): prep pytest for moving to gh-workflows

* ci(pytest): test with caching off

* ci(pytest): remove redundant var build_test_img

* ci(pytest): fix "" workflow syntax --> ''

* ci(pytest): quote ci-development fallback

* ci(pytest): move where default ci-development tag is set

* ci(pytest): reenable caching for workflow images

* ci(pytest): inputs.cache_imgs to array

* ci(pytest): allow passing of image_tag to image build stage

* ci: replace all reusable workflows with hotosm/gh-workflows

* ci(pytest): use new format of reusable pytest workflow

* build: rename API_TAG_OVERRIDE --> TAG_OVERRIDE

.env.example

@@ -1,7 +1,6 @@
 ### copy to .env and set variables
 
 ### ODK Central ###
-ODK_CENTRAL_VERSION=v2023.2.1
 ODK_CENTRAL_URL=https://central-proxy
 ODK_CENTRAL_USER=[email protected]
 ODK_CENTRAL_PASSWD=testuserpassword

.github/labeler.yml

@@ -9,7 +9,6 @@
   - "**/Dockerfile"
   - "**/*.dockerfile"
   - "**/*entrypoint.sh"
-  - "josm/**/*"
 "migration":
   - "src/backend/migrations/**/*"
 "documentation":
@@ -20,3 +19,5 @@
   - "INSTALL.md"
 "ODK":
   - "odkcentral/**/*"
+"contrib":
+  - "contrib/**/*"

.github/workflows/build_and_deploy.yml

@@ -1,3 +1,5 @@
+# Workflow for build and auto-deploy of branches
+
 name: Build and Deploy
 
 on:
@@ -15,52 +17,47 @@ on:
 
 jobs:
   pytest:
-    uses: ./.github/workflows/r-pytest.yml
+    uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main
     with:
-      image_tag: ci-${{ github.ref_name }}
+      image_name: ghcr.io/${{ github.repository }}/backend
+      build_context: src/backend
+      build_args: |
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}
+      docker_compose_service: api
+      tag_override: ci-${{ github.ref_name }}
     secrets: inherit
 
   frontend-tests:
-    uses: ./.github/workflows/r-frontend_tests.yml
-
-  extract-vars:
-    needs:
-      - pytest
-      - frontend-tests
-    uses: ./.github/workflows/r-extract_vars.yml
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main
     with:
-      environment: ${{ github.ref_name }}
+      working_dir: src/frontend
 
   backend-build:
     uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main
-    needs: [extract-vars]
     with:
       context: src/backend
       build_target: prod
-      image_tags: |
-        "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-vars.outputs.api_version }}-${{ github.ref_name }}"
-        "ghcr.io/hotosm/fmtm/backend:latest"
+      image_name: ghcr.io/${{ github.repository }}/backend
       build_args: |
-        APP_VERSION=${{ needs.extract-vars.outputs.api_version }}
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}
 
   frontend-main-build:
     uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main
-    needs: [extract-vars]
     with:
       context: src/frontend
       dockerfile: prod.dockerfile
       build_target: prod
-      image_tags: |
-        "ghcr.io/hotosm/fmtm/frontend:${{ needs.extract-vars.outputs.frontend_main_version }}-${{ github.ref_name }}"
-        "ghcr.io/hotosm/fmtm/frontend:latest"
+      image_name: ghcr.io/${{ github.repository }}/frontend
       build_args: |
-        APP_VERSION=${{ needs.extract-vars.outputs.frontend_main_version }}
-        VITE_API_URL=${{ needs.extract-vars.outputs.api_url }}
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}
+        VITE_API_URL=${{ vars.URL_SCHEME }}://${{ vars.API_URL }}"
 
   smoke-test-backend:
     runs-on: ubuntu-latest
     needs:
-      - extract-vars
       - backend-build
     environment:
       name: ${{ github.ref_name }}
@@ -69,55 +66,28 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Environment to .env
+      - name: Vars and Secrets to Env
         env:
           GIT_BRANCH: ${{ github.ref_name }}
-          API_VERSION: ${{ needs.extract-vars.outputs.api_version }}
-          FRONTEND_MAIN_VERSION: ${{ needs.extract-vars.outputs.frontend_main_version }}
+          VARS_CONTEXT: ${{ toJson(vars) }}
+          SECRETS_CONTEXT: ${{ toJson(secrets) }}
         run: |
-          echo "${{ secrets.DOTENV }}" > .env
-          echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
-          echo "API_VERSION=${API_VERSION}" >> .env
-          echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env
+          to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; }
+
+          echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV
+          echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV
+          echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV
+
+      - name: Create .env file
+        run: env > .env
 
       - name: Backend smoke test
         run: |
-          echo "Not implemented" 
-          # source .env
-          # docker network create fmtm
-
-          # docker pull "postgis/postgis:14-3.3-alpine"
-          # docker run --rm -d \
-          #   --name=fmtm-db \
-          #   --network=fmtm \
-          #   -e POSTGRES_USER=fmtm \
-          #   -e POSTGRES_PASSWORD=fmtm \
-          #   -e POSTGRES_DB=fmtm \
-          #   "postgis/postgis:14-3.3-alpine"
-
-          # docker pull "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
-          # docker run --rm -d \
-          #   --network=fmtm \
-          #   -p 8080:8080 \
-          #   -e OSM_CLIENT_ID="test" \
-          #   -e OSM_CLIENT_SECRET="test" \
-          #   -e OSM_SECRET_KEY="test" \
-          #   -e S3_ACCESS_KEY="fmtm" \
-          #   -e S3_SECRET_KEY="somelongpassword" \
-          #   "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
-
-          # # First wait 10 seconds for API
-          # sleep 10
-          # # Check the exit status of curl and exit the job if it fails
-          # if ! curl -f http://localhost:8080/docs; then
-          #   echo "curl failed to access http://localhost:8080/docs"
-          #   exit 1
-          # fi
+          docker compose up -d api --wait --wait-timeout 60
 
   smoke-test-frontend:
     runs-on: ubuntu-latest
     needs:
-      - extract-vars
       - frontend-main-build
     environment:
       name: ${{ github.ref_name }}
@@ -130,40 +100,11 @@ jobs:
         run: echo "Not implemented"
 
   deploy-containers:
-    runs-on: ubuntu-latest
     needs:
-      - extract-vars
       - smoke-test-backend
       - smoke-test-frontend
-    environment:
-      name: ${{ github.ref_name }}
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Environment to .env
-        env:
-          GIT_BRANCH: ${{ github.ref_name }}
-          API_VERSION: ${{ needs.extract-vars.outputs.api_version }}
-          FRONTEND_MAIN_VERSION: ${{ needs.extract-vars.outputs.frontend_main_version }}
-        run: |
-          echo "${{ secrets.DOTENV }}" > .env
-          echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
-          echo "API_VERSION=${API_VERSION}" >> .env
-          echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env
-
-      - uses: webfactory/[email protected]
-        with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Disable Host key verification
-        # Hack to prevent "Host key verification failed". Should be replaced with a ssh-keyscan based solution
-        run: echo "StrictHostKeyChecking no" >> ~/.ssh/config
-
-      - name: Deploy
-        run: |
-          docker compose --file "docker-compose.${{ github.ref_name }}.yml" pull
-          docker compose --file "docker-compose.${{ github.ref_name }}.yml" up --detach --remove-orphans
-        env:
-          DOCKER_HOST: ${{ vars.DOCKER_HOST }}
+    uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@main
+    with:
+      environment: ${{ github.ref_name }}
+      docker_compose_file: "docker-compose.${{ github.ref_name }}.yml"
+    secrets: inherit

.github/workflows/build_ci_img.yml

@@ -15,17 +15,13 @@ on:
   workflow_dispatch:
 
 jobs:
-  extract-vars:
-    uses: ./.github/workflows/r-extract_vars.yml
-
   backend-ci-build:
     uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main
-    needs: [extract-vars]
     with:
       context: src/backend
       build_target: ci
       image_tags: |
-        "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-vars.outputs.api_version }}-ci-${{ github.ref_name }}"
-        "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}"
+        "ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}"
       build_args: |
-        APP_VERSION=${{ needs.extract-vars.outputs.api_version }}
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}

.github/workflows/build_odk_imgs.yml

@@ -17,15 +17,15 @@ jobs:
     with:
       context: odkcentral/api
       image_tags: |
-        "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_VERSION }}"
-        "ghcr.io/hotosm/fmtm/odkcentral:latest"
+        "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}"
+        "ghcr.io/${{ github.repository }}/odkcentral:latest"
       build_args: |
-        ODK_CENTRAL_VERSION=${{ vars.ODK_CENTRAL_VERSION }}
+        ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }}
 
   build-proxy:
     uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main
     with:
       context: odkcentral/proxy
       image_tags: |
-        "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_VERSION }}"
-        "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
+        "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}"
+        "ghcr.io/${{ github.repository }}/odkcentral-proxy:latest"

.github/workflows/docs.yml

@@ -6,37 +6,29 @@ on:
       - docs/**
       - src/**
       - mkdocs.yml
-    branches: [main]
+    branches: [development]
   # Allow manual trigger (workflow_dispatch)
   workflow_dispatch:
 
 jobs:
   build_doxygen:
     uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@main
     with:
-      cache_paths: |
-        docs/apidocs
-        docs/openapi.json
-      cache_key: docs-build
+      output_path: docs/apidocs
 
   build_openapi_json:
     uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@main
     with:
-      image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}
+      image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
       example_env_file_path: ".env.example"
-      cache_paths: |
-        docs/apidocs
-        docs/openapi.json
-      cache_key: docs-build
+      output_path: docs/openapi.json
 
   publish_docs:
     uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@main
     needs:
       - build_doxygen
       - build_openapi_json
     with:
-      image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}
-      cache_paths: |
-        docs/apidocs
-        docs/openapi.json
-      cache_key: docs-build
+      image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
+      doxygen: true
+      openapi: true

.github/workflows/pr_test_backend.yml

@@ -0,0 +1,31 @@
+name: PR Test Backend
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+      - development
+    # Workflow is triggered only if src/backend changes
+    paths:
+      - src/backend/**
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+jobs:
+  pytest:
+    uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main
+    with:
+      image_name: ghcr.io/${{ github.repository }}/backend
+      build_context: src/backend
+      build_args: |
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}
+      docker_compose_service: api
+      cache_extra_imgs: |
+        "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}"
+        "docker.io/minio/minio:${{ vars.MINIO_TAG }}"
+      # For caching odk central images, add:
+      # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}"
+      # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}"
+    secrets: inherit

.github/workflows/pr_test_frontend.yml

@@ -0,0 +1,19 @@
+name: PR Tests Frontend
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+      - development
+    # Workflow is triggered only if src/frontend changes
+    paths:
+      - src/frontend/**
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+jobs:
+  frontend-tests:
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main
+    with:
+      working_dir: src/frontend