From d50cdff50a72a25efe31eb85c45dd69ed494ee59 Mon Sep 17 00:00:00 2001 From: Thomas Vaillant Date: Wed, 30 Oct 2024 15:16:49 +0100 Subject: [PATCH] ci: factorize CI with reusable workflows --- .github/workflows/on-merge-to-develop.yml | 86 ++++--------------- .github/workflows/on-release-tag-alpha.yml | 27 +++--- .github/workflows/on-release-tag.yml | 63 +++++--------- .github/workflows/pull-request-checks.yml | 65 +++----------- .github/workflows/reusable-e2e-tests.yml | 42 +++++++++ .../reusable-get-log4brains-version.yml | 22 +++++ ...eusable-load-nodejs-supported-versions.yml | 22 +++++ .github/workflows/reusable-quality-checks.yml | 24 ++++++ .github/workflows/reusable-tests.yml | 63 ++++++++++++++ ...e-wait-for-npm-version-to-be-published.yml | 43 ++++++++++ .../workflows/scheduled-weekly-e2e-stable.yml | 30 ++----- README.md | 4 +- 12 files changed, 285 insertions(+), 206 deletions(-) create mode 100644 .github/workflows/reusable-e2e-tests.yml create mode 100644 .github/workflows/reusable-get-log4brains-version.yml create mode 100644 .github/workflows/reusable-load-nodejs-supported-versions.yml create mode 100644 .github/workflows/reusable-quality-checks.yml create mode 100644 .github/workflows/reusable-tests.yml create mode 100644 .github/workflows/reusable-wait-for-npm-version-to-be-published.yml diff --git a/.github/workflows/on-merge-to-develop.yml b/.github/workflows/on-merge-to-develop.yml index 0efca4e..d566e43 100644 --- a/.github/workflows/on-merge-to-develop.yml +++ b/.github/workflows/on-merge-to-develop.yml @@ -16,88 +16,38 @@ defaults: shell: bash jobs: - quality: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc # normally the current LTS - cache: yarn - cache-dependency-path: yarn.lock - - run: yarn install --frozen-lockfile - - # TODO: make dev & test work without having to build everything (inspiration: https://github.com/Izhaki/mono.ts) - # - run: yarn typescript - - run: yarn build - - run: yarn format - - run: yarn lint + quality-checks: + uses: ./.github/workflows/reusable-quality-checks.yml load-nodejs-supported-versions: - runs-on: ubuntu-latest - outputs: - node_versions: ${{ steps.get-versions.outputs.node_versions }} - steps: - - name: Get supported Node.js versions from JSON - id: get-versions - run: | - node_versions=$(cat .github/supported_nodejs_versions.json | jq -r '.versions | @json') - echo "::set-output name=node_versions::$node_versions" + uses: ./.github/workflows/reusable-load-nodejs-supported-versions.yml - # We test the current LTS on all OSes and other supported versions only on Ubuntu for performance reasons - # All the other os/version tuples will be tested in the post-release job + # We test the current LTS on all OSes and other supported versions are only tested on Ubuntu for performance reasons + # All the other os/version tuples will be tested in the post-release E2E workflow tests-per-nodejs-version: needs: load-nodejs-supported-versions strategy: matrix: node-version: ${{ fromJson(needs.load-supported-versions.outputs.node_versions) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetch all history to make Jest snapshot tests work - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: yarn - cache-dependency-path: yarn.lock - - run: yarn install --frozen-lockfile - - run: yarn build - - run: yarn test - - name: E2E tests - run: | - yarn link-cli - echo "$(yarn global bin)" >> $GITHUB_PATH - yarn e2e + uses: ./.github/workflows/reusable-tests.yml + with: + os: ubuntu-latest + node-version: ${{ matrix.node-version }} - tests-per-other-os: + tests-LTS-per-other-os: strategy: matrix: os: [windows-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetch all history to make Jest snapshot tests work - - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc # normally the current LTS - cache: yarn - cache-dependency-path: yarn.lock - - run: yarn install --frozen-lockfile - - run: yarn build - - run: yarn test - - name: E2E tests - run: | - yarn link-cli - echo "$(yarn global bin)" >> $GITHUB_PATH - yarn e2e + uses: ./.github/workflows/reusable-tests.yml + with: + os: ${{ matrix.os }} + node-version-file: .nvmrc # current LTS publish-pages: needs: - - quality + - quality-checks - tests-per-nodejs-version - - tests-per-other-os + - tests-LTS-per-other-os runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -106,7 +56,7 @@ jobs: fetch-depth: 0 # required by Log4brains to work correctly (needs the whole Git history) - uses: actions/setup-node@v4 with: - node-version-file: .nvmrc # normally the current LTS + node-version-file: .nvmrc # current LTS cache: yarn cache-dependency-path: yarn.lock - run: yarn install --frozen-lockfile @@ -139,7 +89,7 @@ jobs: run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* || true - uses: actions/setup-node@v4 with: - node-version-file: .nvmrc # normally the current LTS + node-version-file: .nvmrc # current LTS cache: yarn cache-dependency-path: yarn.lock registry-url: https://registry.npmjs.org/ # needed by lerna to push to NPM diff --git a/.github/workflows/on-release-tag-alpha.yml b/.github/workflows/on-release-tag-alpha.yml index 756922b..c86aee5 100644 --- a/.github/workflows/on-release-tag-alpha.yml +++ b/.github/workflows/on-release-tag-alpha.yml @@ -8,7 +8,18 @@ on: - v*-alpha.* jobs: + get-log4brains-version: + uses: ./.github/workflows/reusable-get-log4brains-version.yml + + wait-for-npm-version-to-be-published: + needs: get-log4brains-version + uses: ./.github/workflows/reusable-wait-for-npm-version-to-be-published.yml + with: + npm-package: log4brains + npm-version: ${{ needs.get-log4brains-version.outputs.version }} + build-and-push: + needs: wait-for-npm-version-to-be-published runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -17,18 +28,4 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push the latest alpha Docker image - run: | - export MDF_BRANCH_TAG=alpha - i=0 - while ! make -C docker release version-tag branch-tag - do - if [[ ${i} -gt 20 ]] - then - echo "Failure for more than 10 minutes -> abort" - exit 1 - fi - - echo "Maybe the package is not available yet on NPM. Let's wait 30s..." - sleep 30 - ((i=i+1)) - done + run: MDF_BRANCH_TAG=alpha make -C docker release version-tag branch-tag diff --git a/.github/workflows/on-release-tag.yml b/.github/workflows/on-release-tag.yml index 077d607..df926a1 100644 --- a/.github/workflows/on-release-tag.yml +++ b/.github/workflows/on-release-tag.yml @@ -8,51 +8,30 @@ on: - v* jobs: + get-log4brains-version: + uses: ./.github/workflows/reusable-get-log4brains-version.yml + + wait-for-npm-version-to-be-published: + needs: get-log4brains-version + uses: ./.github/workflows/reusable-wait-for-npm-version-to-be-published.yml + with: + npm-package: log4brains + npm-version: ${{ needs.get-log4brains-version.outputs.version }} + load-nodejs-supported-versions: - runs-on: ubuntu-latest - outputs: - node_versions: ${{ steps.get-versions.outputs.node_versions }} - steps: - - name: Get supported Node.js versions from JSON - id: get-versions - run: | - node_versions=$(cat .github/supported_nodejs_versions.json | jq -r '.versions | @json') - echo "::set-output name=node_versions::$node_versions" + uses: ./.github/workflows/reusable-load-nodejs-supported-versions.yml - e2e: - needs: load-nodejs-supported-versions + e2e-tests: + needs: + - load-nodejs-supported-versions + - get-log4brains-version + - wait-for-npm-version-to-be-published strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] node-version: ${{ fromJson(needs.load-supported-versions.outputs.node_versions) }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # required by Log4brains to work correctly (needs the whole Git history) - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - name: Run npm install - run: | - log4brains_version="$(cat lerna.json | jq -r .version)" - - i=0 - while ! npm install -g log4brains@${log4brains_version} - do - if [[ ${i} -gt 20 ]] - then - echo "Failure for more than 10 minutes -> abort" - exit 1 - fi - - echo "Maybe the package is not available yet on NPM. Let's wait 30s..." - sleep 30 - ((i=i+1)) - done - - name: Tests # TODO: test the preview and add some assertions :-) For now we just check the exit codes. Or manage to run e2e-tests/e2e-launcher.js from here - run: | - log4brains --version - log4brains init --defaults - log4brains adr list - log4brains build + uses: ./.github/workflows/reusable-e2e-tests.yml + with: + npm-package-fullname: log4brains@${{ needs.get-log4brains-version.outputs.version }} + os: ${{ matrix.os }} + node-version: ${{ matrix.node-version }} diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 7c18e90..a261e87 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -9,62 +9,17 @@ defaults: shell: bash jobs: - quality: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc # normally the current LTS - cache: yarn - cache-dependency-path: yarn.lock - - run: yarn install --frozen-lockfile + quality-checks: + uses: ./.github/workflows/reusable-quality-checks.yml - # TODO: make dev & test work without having to build everything (inspiration: https://github.com/Izhaki/mono.ts) - # - run: yarn typescript - - run: yarn build - - run: yarn format - - run: yarn lint - - tests-per-os: + # We only test the LTS on all OSes for performance reasons + # Other versions will be tested on the main workflow after the merge + tests-LTS-per-os: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetch all history to make Jest snapshot tests work - - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc # normally the current LTS; we test other versions in the main Build workflow because it's too slow - cache: yarn - cache-dependency-path: yarn.lock - - run: yarn install --frozen-lockfile - - - name: Check for yarn.lock changes - id: yarn-lock - run: git diff --quiet origin/develop HEAD -- yarn.lock - continue-on-error: true - # - steps.yarn-lock.outcome == 'success' --> yarn.lock was not changed - # - steps.yarn-lock.outcome == 'failure' --> yarn.lock was changed - - # We have to build all the packages before the tests - # Because init-log4brains's integration tests use @log4brains/cli, which uses @log4brains/core - # TODO: we should separate tests that require built packages of the others, to get a quicker feedback - # Once it's done, we should add "yarn test" in each package's preVersion script - - run: yarn build - - - name: Test changed packages - if: ${{ steps.yarn-lock.outcome == 'success' }} - run: yarn test --since origin/develop - - - name: Test all packages (because yarn.lock has changed) - if: ${{ steps.yarn-lock.outcome == 'failure' }} - run: yarn test - - - name: E2E tests - run: | - yarn link-cli - echo "$(yarn global bin)" >> $GITHUB_PATH - yarn e2e + uses: ./.github/workflows/reusable-tests.yml + with: + only-changed-packages: true # for performance + os: ${{ matrix.os }} + node-version-file: .nvmrc # current LTS diff --git a/.github/workflows/reusable-e2e-tests.yml b/.github/workflows/reusable-e2e-tests.yml new file mode 100644 index 0000000..64e265a --- /dev/null +++ b/.github/workflows/reusable-e2e-tests.yml @@ -0,0 +1,42 @@ +on: + workflow_call: + inputs: + npm-package-fullname: + required: true + type: string + os: + required: true + type: string + # Set only one of these two inputs: + node-version: + required: false + type: string + node-version-file: + required: false + type: string + +defaults: + run: + shell: bash + +jobs: + e2e-tests: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # required by Log4brains to work correctly (needs the whole Git history) + - uses: actions/setup-node@v4 + with: + node-version-file: ${{ inputs.node-version-file }} + node-version: ${{ inputs.node-version }} + + - run: npm install -g ${{ inputs.npm-package-fullname }} + + # TODO: test the preview and add some assertions :-) For now we just check the exit codes. Or manage to run e2e-tests/e2e-launcher.js from here + - name: Tests + run: | + log4brains --version + log4brains init --defaults + log4brains adr list + log4brains build diff --git a/.github/workflows/reusable-get-log4brains-version.yml b/.github/workflows/reusable-get-log4brains-version.yml new file mode 100644 index 0000000..c5ad221 --- /dev/null +++ b/.github/workflows/reusable-get-log4brains-version.yml @@ -0,0 +1,22 @@ +on: + workflow_call: + outputs: + version: + description: "Log4brains current version" + value: ${{ jobs.get-log4brains-version.outputs.version }} + +defaults: + run: + shell: bash + +jobs: + get-log4brains-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get-version.outputs.version }} + steps: + - name: Get log4brains version from lerna.json + id: get-version + run: | + version="$(cat lerna.json | jq -r .version)" + echo "::set-output name=version::$version" diff --git a/.github/workflows/reusable-load-nodejs-supported-versions.yml b/.github/workflows/reusable-load-nodejs-supported-versions.yml new file mode 100644 index 0000000..4c91d76 --- /dev/null +++ b/.github/workflows/reusable-load-nodejs-supported-versions.yml @@ -0,0 +1,22 @@ +on: + workflow_call: + outputs: + node_versions: + description: "Node.js versions officially supported by the project" + value: ${{ jobs.load-nodejs-supported-versions.outputs.node_versions }} + +defaults: + run: + shell: bash + +jobs: + load-nodejs-supported-versions: + runs-on: ubuntu-latest + outputs: + node_versions: ${{ steps.get-versions.outputs.node_versions }} + steps: + - name: Get supported Node.js versions from JSON file + id: get-versions + run: | + node_versions=$(cat .github/supported_nodejs_versions.json | jq -r '.versions | @json') + echo "::set-output name=node_versions::$node_versions" diff --git a/.github/workflows/reusable-quality-checks.yml b/.github/workflows/reusable-quality-checks.yml new file mode 100644 index 0000000..3d61982 --- /dev/null +++ b/.github/workflows/reusable-quality-checks.yml @@ -0,0 +1,24 @@ +on: + workflow_call: + +defaults: + run: + shell: bash + +jobs: + quality-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc # current LTS + cache: yarn + cache-dependency-path: yarn.lock + - run: yarn install --frozen-lockfile + + # TODO: make dev & test work without having to build everything (inspiration: https://github.com/Izhaki/mono.ts) + # - run: yarn typescript + - run: yarn build + - run: yarn format + - run: yarn lint diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml new file mode 100644 index 0000000..7959ead --- /dev/null +++ b/.github/workflows/reusable-tests.yml @@ -0,0 +1,63 @@ +on: + workflow_call: + inputs: + only-changed-packages: + required: false + type: boolean + default: false # by default it will test all packages; also if only-changed-packages=true and yarn.lock has changed: it will test all packages too + os: + required: true + type: string + # Set only one of these two inputs: + node-version: + required: false + type: string + node-version-file: + required: false + type: string + +defaults: + run: + shell: bash + +jobs: + tests: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch all history to make Jest snapshot tests work + - uses: actions/setup-node@v4 + with: + node-version-file: ${{ inputs.node-version-file }} + node-version: ${{ inputs.node-version }} + cache: yarn + cache-dependency-path: yarn.lock + - run: yarn install --frozen-lockfile + + - name: Check for yarn.lock changes + id: yarn-lock + run: git diff --quiet origin/develop HEAD -- yarn.lock + continue-on-error: true + # - steps.yarn-lock.outcome == 'success' --> yarn.lock was not changed + # - steps.yarn-lock.outcome == 'failure' --> yarn.lock was changed + + # We have to build all the packages before the tests + # Because init-log4brains's integration tests use @log4brains/cli, which uses @log4brains/core + # TODO: we should separate tests that require built packages of the others, to get a quicker feedback + # Once it's done, we should add "yarn test" in each package's preVersion script + - run: yarn build + + - name: Test only packages that have changed compared to the main branch + if: ${{ inputs.only-changed-packages && steps.yarn-lock.outcome == 'success' }} + run: yarn test --since origin/develop + + - name: Test all packages (only-changed-packages=false or yarn.lock has changed) + if: ${{ !inputs.only-changed-packages || steps.yarn-lock.outcome == 'failure' }} + run: yarn test + + - name: E2E tests + run: | + yarn link-cli + echo "$(yarn global bin)" >> $GITHUB_PATH + yarn e2e diff --git a/.github/workflows/reusable-wait-for-npm-version-to-be-published.yml b/.github/workflows/reusable-wait-for-npm-version-to-be-published.yml new file mode 100644 index 0000000..75185e2 --- /dev/null +++ b/.github/workflows/reusable-wait-for-npm-version-to-be-published.yml @@ -0,0 +1,43 @@ +on: + workflow_call: + inputs: + npm-package: + required: true + type: string + npm-version: # only exact versions allowed, not tags like "latest" or "alpha" + required: true + type: string + wait-minutes: + required: false + type: string + default: "10" + +defaults: + run: + shell: bash + +jobs: + wait-for-npm-version-to-be-published: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-node@v4 + - name: Wait + run: | + echo "Checking if ${{ inputs.npm-package }}@${{ inputs.npm-version }} is available on NPM (waiting max ${{ inputs.wait-minutes }} min)..." + i=0 + while ! npm view ${{ inputs.npm-package }} versions --json | jq -e 'index("${{ inputs.npm-version }}")' &> /dev/null + do + if [[ ${i} -gt ${{ inputs.wait-minutes }} ]] + then + echo "Failure for more than ${{ inputs.wait-minutes }} minutes" + echo "Available versions:" + npm view ${{ inputs.npm-package }} versions --json + echo "Abort" + exit 1 + fi + + echo "${{ inputs.npm-package }}@${{ inputs.npm-version }} is not available yet on NPM. Let's wait 60s..." + sleep 60 + ((i=i+1)) + done + echo "${{ inputs.npm-package }}@${{ inputs.npm-version }} is available" diff --git a/.github/workflows/scheduled-weekly-e2e-stable.yml b/.github/workflows/scheduled-weekly-e2e-stable.yml index 62e7a9e..4188f1a 100644 --- a/.github/workflows/scheduled-weekly-e2e-stable.yml +++ b/.github/workflows/scheduled-weekly-e2e-stable.yml @@ -9,15 +9,7 @@ on: jobs: load-nodejs-supported-versions: - runs-on: ubuntu-latest - outputs: - node_versions: ${{ steps.get-versions.outputs.node_versions }} - steps: - - name: Get supported Node.js versions from JSON - id: get-versions - run: | - node_versions=$(cat .github/supported_nodejs_versions.json | jq -r '.versions | @json') - echo "::set-output name=node_versions::$node_versions" + uses: ./.github/workflows/reusable-load-nodejs-supported-versions.yml e2e: needs: load-nodejs-supported-versions @@ -25,18 +17,8 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] node-version: ${{ fromJson(needs.load-supported-versions.outputs.node_versions) }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # required by Log4brains to work correctly (needs the whole Git history) - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - run: npm install -g log4brains - - name: Tests # TODO: test the preview and add some assertions :-) For now we just check the exit codes. Or manage to run e2e-tests/e2e-launcher.js from here - run: | - log4brains --version - log4brains init --defaults - log4brains adr list - log4brains build + uses: ./.github/workflows/reusable-e2e-tests.yml + with: + npm-package-fullname: log4brains # stable version + os: ${{ matrix.os }} + node-version: ${{ matrix.node-version }} diff --git a/README.md b/README.md index 39dd586..95d3052 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ License - - Build Status + + Build Status log4brains latest version