From c1bf1324c53f3deb255beac67e258c181ee241b6 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 11 Nov 2024 13:35:42 -0600 Subject: [PATCH 1/8] pants ci: Consolidate env var definitions (DRY) --- .github/workflows/test.yaml | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9be21ee106..3314940b34 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,6 +22,16 @@ on: # # run every night at midnight # - cron: '0 0 * * *' +env: + COLUMNS: '120' + + # GitHub Actions uses the 'runner' user, so use that instead of stanley. + ST2TESTS_SYSTEM_USER: 'runner' + + # This is the host:port provided by services.redis + ST2TESTS_REDIS_HOST: '127.0.0.1' + ST2TESTS_REDIS_PORT: '6379' + jobs: pants-plugins-tests: name: 'Pants Plugins Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}' @@ -63,9 +73,6 @@ jobs: ports: - 6379:6379/tcp - env: - COLUMNS: '120' - steps: - name: Checkout repository uses: actions/checkout@v4 @@ -99,11 +106,6 @@ jobs: gha-cache-key: cache0-py${{ matrix.python.version }} - name: Test pants-plugins - env: - # Github Actions uses the 'runner' user, so use that instead of stanley. - ST2TESTS_SYSTEM_USER: 'runner' - ST2TESTS_REDIS_HOST: '127.0.0.1' - ST2TESTS_REDIS_PORT: '6379' run: | pants test pants-plugins/:: @@ -161,9 +163,6 @@ jobs: ports: - 6379:6379/tcp - env: - COLUMNS: '120' - steps: - name: Checkout repository uses: actions/checkout@v4 @@ -197,11 +196,6 @@ jobs: gha-cache-key: cache0-py${{ matrix.python.version }} - name: Unit Tests - env: - # Github Actions uses the 'runner' user, so use that instead of stanley. - ST2TESTS_SYSTEM_USER: 'runner' - ST2TESTS_REDIS_HOST: '127.0.0.1' - ST2TESTS_REDIS_PORT: '6379' run: > pants --python-bootstrap-search-path=[] @@ -257,9 +251,6 @@ jobs: ports: - 6379:6379/tcp - env: - COLUMNS: '120' - steps: - name: Checkout repository uses: actions/checkout@v4 @@ -293,11 +284,6 @@ jobs: gha-cache-key: cache0-py${{ matrix.python.version }} - name: Pack Tests - env: - # Github Actions uses the 'runner' user, so use that instead of stanley. - ST2TESTS_SYSTEM_USER: 'runner' - ST2TESTS_REDIS_HOST: '127.0.0.1' - ST2TESTS_REDIS_PORT: '6379' run: > pants --python-bootstrap-search-path=[] From 4e0fd655823a76d25971b580fcbfb12f80c0a334 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 6 Nov 2024 18:05:46 -0600 Subject: [PATCH 2/8] pants ci: add ST2_CI var to enable ci-only tests --- .github/workflows/test.yaml | 3 +++ pants.toml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3314940b34..37f71ae119 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,6 +25,9 @@ on: env: COLUMNS: '120' + # Tell StackStorm that we are indeed in CI mode, using our CI-provider agnostic var. + ST2_CI: 'true' + # GitHub Actions uses the 'runner' user, so use that instead of stanley. ST2TESTS_SYSTEM_USER: 'runner' diff --git a/pants.toml b/pants.toml index 581ba5b87b..f6ca591fd6 100644 --- a/pants.toml +++ b/pants.toml @@ -257,6 +257,9 @@ extra_env_vars = [ # "ST2_COORDINATION__URL", # Tests will override this with one of: # "redis://{ST2TESTS_REDIS_HOST}:{ST2TESTS_REDIS_PORT}?namespace=_st2_test{ST2TESTS_PARALLEL_SLOT} # "zake://" + # CI-specific vars + "ST2_CI", + "ST2_CI_RUN_ORQUESTA_PAUSE_RESUME_TESTS", ] # 10 min should be more than enough even for integration tests. timeout_default = 600 # seconds From a7cd72dd6b44d1c14531ced858a836a35745a91d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 1 Nov 2024 22:39:44 -0500 Subject: [PATCH 3/8] pants ci: add integration tests job pants `--tag=...` notes: `--tags=abc,def` (a csv list): is an OR condition (either `abc` tag, or `def` tag, or both). `--tags=abc --tag=def` (multiple entries): is an AND condition (Both `abc` tag AND `def` tag). So, we use `--tag=integration --tag=-st2cluster` to select targets that have the `integration` tag AND do NOT have the `st2cluster` tag. --- .github/workflows/test.yaml | 90 ++++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 37f71ae119..9edb0616ec 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -301,7 +301,93 @@ jobs: path: .pants.d/pants.log if: always() # We want the log even on failures. - #integration-tests: TODO: run integration tests + integration-tests: + name: 'Integration Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}' + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + # NOTE: We need to use full Python version as part of Python deps cache key otherwise + # setup virtualenv step will fail. + python: + - {version-short: '3.8', version: '3.8.10'} + - {version-short: '3.9', version: '3.9.14'} + + services: + mongo: + image: mongo:7.0 + ports: + - 27017:27017 + + rabbitmq: + image: rabbitmq:3.8-management + options: >- + --name rabbitmq + ports: + - 5671:5671/tcp # AMQP SSL port + - 5672:5672/tcp # AMQP standard port + - 15672:15672/tcp # Management: HTTP, CLI + + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --name "redis" + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379/tcp + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # a test uses a submodule, and pants needs access to it to calculate deps. + submodules: 'recursive' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules + run: | + git submodule status + git submodule foreach 'git fetch --all --tags' + git submodule foreach 'git tag' + + - name: 'Set up Python (${{ matrix.python.version }})' + id: python + uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.python.version }}' + + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages + + - name: Initialize Pants and its GHA caches + uses: ./.github/actions/init-pants + with: + # To ignore a bad cache, bump the cache* integer. + gha-cache-key: cache0-py${{ matrix.python.version }} + + - name: Integration Tests + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + --tag=integration + --tag=-st2cluster + test '::' + + - name: Upload pants log + uses: actions/upload-artifact@v4 + with: + name: pants-log-py${{ matrix.python.version }}-integration-tests + path: .pants.d/pants.log + if: always() # We want the log even on failures. set_merge_ok: name: Set Merge OK (Tests) @@ -310,7 +396,7 @@ jobs: - pants-plugins-tests - unit-tests - pack-tests - #- integration-tests + - integration-tests outputs: merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }} runs-on: ubuntu-latest From 8fb0deb545bb62b1783cfd112b911f265e1bd7cc Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 5 Nov 2024 21:32:51 -0600 Subject: [PATCH 4/8] pants ci: orquesta (uses=st2cluster) itests pants `--tag=...` notes: `--tags=abc,def` (a csv list): is an OR condition (either `abc` tag, or `def` tag, or both). `--tags=abc --tag=def` (multiple entries): is an AND condition (Both `abc` tag AND `def` tag). So, we use `--tag=integration --tag=st2cluster` to select targets that have the `integration tag AND the `st2cluster` tag. (The other integration test job excludes the st2cluster tagged targets). --- .github/workflows/test.yaml | 119 ++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9edb0616ec..641c402c18 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -389,6 +389,124 @@ jobs: path: .pants.d/pants.log if: always() # We want the log even on failures. + integration-st2cluster-tests: + name: 'Integration Tests (Full Cluster)- Python ${{ matrix.python.version-short }}' + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + # NOTE: We need to use full Python version as part of Python deps cache key otherwise + # setup virtualenv step will fail. + python: + - {version-short: '3.8', version: '3.8.10'} + - {version-short: '3.9', version: '3.9.14'} + + services: + mongo: + image: mongo:7.0 + ports: + - 27017:27017 + + rabbitmq: + image: rabbitmq:3.8-management + options: >- + --name rabbitmq + ports: + - 5671:5671/tcp # AMQP SSL port + - 5672:5672/tcp # AMQP standard port + - 15672:15672/tcp # Management: HTTP, CLI + + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --name "redis" + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379/tcp + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # a test uses a submodule, and pants needs access to it to calculate deps. + submodules: 'recursive' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules + run: | + git submodule status + git submodule foreach 'git fetch --all --tags' + git submodule foreach 'git tag' + + - name: 'Set up Python (${{ matrix.python.version }})' + id: python + uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.python.version }}' + + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages + + - name: Initialize Pants and its GHA caches + uses: ./.github/actions/init-pants + with: + # To ignore a bad cache, bump the cache* integer. + gha-cache-key: cache0-py${{ matrix.python.version }} + + - name: Export virtualenv to run Dev ST2 Cluster + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + export + --resolve=st2 + + - name: Launch Dev ST2 Cluster + env: + VIRTUALENV_DIR: ./dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }} + run: | + sudo -E ./scripts/github/prepare-integration.sh + + - name: Integration Tests + env: + ST2_CI_RUN_ORQUESTA_PAUSE_RESUME_TESTS: 'true' + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + --tag=integration + --tag=st2cluster + test '::' + + - name: Compress Service Logs Before upload + if: ${{ failure() }} + run: | + ./tools/launchdev.sh stop # stop st2 before collecting logs + tar cvzpf logs.tar.gz logs/* + + - name: Upload StackStorm services Logs + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: logs-py${{ matrix.python-version }}-st2cluster-integration-tests + path: logs.tar.gz + retention-days: 7 + + - name: Upload pants log + uses: actions/upload-artifact@v4 + with: + name: pants-log-py${{ matrix.python.version }}-st2cluster-integration-tests + path: .pants.d/pants.log + if: always() # We want the log even on failures. + set_merge_ok: name: Set Merge OK (Tests) if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') @@ -397,6 +515,7 @@ jobs: - unit-tests - pack-tests - integration-tests + - integration-st2cluster-tests outputs: merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }} runs-on: ubuntu-latest From 85da1cb5195776312eef641537727e488d6085ce Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 6 Nov 2024 18:42:18 -0600 Subject: [PATCH 5/8] pants ci: copy Reconfigure RabbitMQ task into pants workflow --- .github/workflows/test.yaml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 641c402c18..f81949ad7c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -319,14 +319,34 @@ jobs: ports: - 27017:27017 + # In GHA, these services are started first before the code is checked out. + # We use bitnami images to facilitate reconfiguring RabbitMQ during integration tests. + # We rely on custom config and SSL certs that are in the repo. + # Many images require config in env vars (which we can't change during the test job) + # or they require config in entrypoint args (which we can't override for GHA services) + # bitnami builds ways to get config files from mounted volumes. rabbitmq: - image: rabbitmq:3.8-management + image: bitnami/rabbitmq:3.8 + volumes: + - /home/runner/rabbitmq_conf:/bitnami/conf # RABBITMQ_MOUNTED_CONF_DIR + env: + # tell bitnami/rabbitmq to enable this by default + RABBITMQ_PLUGINS: rabbitmq_management + RABBITMQ_USERNAME: guest + RABBITMQ_PASSWORD: guest + + # These are strictly docker options, not entrypoint args (GHA restriction) options: >- --name rabbitmq ports: + # These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports) + # host_port:container_port/protocol - 5671:5671/tcp # AMQP SSL port - 5672:5672/tcp # AMQP standard port - 15672:15672/tcp # Management: HTTP, CLI + #- 15671:15671/tcp # Management: SSL port + #- 25672:25672/tcp # inter-node or CLI + #- 4369:4369/tcp # epmd redis: # Docker Hub image @@ -373,6 +393,15 @@ jobs: # To ignore a bad cache, bump the cache* integer. gha-cache-key: cache0-py${{ matrix.python.version }} + # This is only required for st2common/tests/integration/test_rabbitmq_ssl_listener.py + - name: Reconfigure RabbitMQ + # bitnami image allows (see bitnami/rabbitmq readme): + # Here we're copying a rabbitmq.config file which won't do anything. + # We need to switch to custom.conf or advanced.config. + timeout-minutes: 2 # may die if rabbitmq fails to start + run: | + ./scripts/github/configure-rabbitmq.sh + - name: Integration Tests run: > pants From 4007f0d1890de6cfc80981f35fbe3e0ee051f109 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 11 Nov 2024 23:10:55 -0600 Subject: [PATCH 6/8] Put bitnami/rabbitmq logs on stdout --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f81949ad7c..b06708f636 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -334,6 +334,7 @@ jobs: RABBITMQ_PLUGINS: rabbitmq_management RABBITMQ_USERNAME: guest RABBITMQ_PASSWORD: guest + RABBITMQ_LOGS: '-' # These are strictly docker options, not entrypoint args (GHA restriction) options: >- From ec6fcb66a00dc56a194190976120ae3e1386959c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 12 Nov 2024 08:19:55 -0600 Subject: [PATCH 7/8] pants ci: fix rabbitmq disk space alert --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b06708f636..50c1f8f265 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -335,6 +335,8 @@ jobs: RABBITMQ_USERNAME: guest RABBITMQ_PASSWORD: guest RABBITMQ_LOGS: '-' + # bitnami's default relative limit was too high + RABBITMQ_DISK_FREE_ABSOLUTE_LIMIT: 50MB # These are strictly docker options, not entrypoint args (GHA restriction) options: >- From 432a6ca7713e6defc8436d1c0861f1682307ac0d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 28 Nov 2024 23:53:43 -0600 Subject: [PATCH 8/8] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 867ff2031b..53a8456f44 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -69,7 +69,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 + #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11