RFC104: add testing for gdal vector pipeline #4157
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Linux Builds | |
| on: | |
| push: | |
| paths-ignore: | |
| - 'doc/**' | |
| - 'docker/**' | |
| branches-ignore: | |
| - 'backport**' | |
| - 'dependabot**' | |
| pull_request: | |
| paths-ignore: | |
| - 'doc/**' | |
| - 'docker/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| linux-build: | |
| # Store the components of the container name as environment variables: | |
| # ${CONTAINER_REGISTRY}/${CONTAINER_REGISTRY_USER}/${CONTAINER_NAME}:${CONTAINER_TAG} | |
| # | |
| # Additionally, CACHE_CONTAINER_TAG may be used as as source for the | |
| # Docker build cache. So if the Dockerfile in a feature branch is | |
| # unchanged relative to master, a full container rebuild should not | |
| # be required. | |
| env: | |
| CONTAINER_REGISTRY: ${{ vars.gdal_container_registry || 'ghcr.io' }} | |
| CONTAINER_REGISTRY_USER: ${{ vars.gdal_container_registry_user || github.repository_owner }} | |
| CONTAINER_NAME: gdal-deps | |
| CONTAINER_TAG: ${{ matrix.container }}-${{ github.base_ref || github.ref_name }} | |
| CACHE_CONTAINER_TAG: ${{ matrix.container }}-master | |
| permissions: | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Matrix variables: | |
| # | |
| # * name : readable summary of configuration, used for display | |
| # * id : used as a ccache key, and to create a build subdirectory | |
| # * container : build environment container and path to build script | |
| # * use_avx2 : if true, determine arch at runtime and use in ccache key | |
| # * build_script : name of custom build script, if any. Will be executed | |
| # inside container, from build subdirectory. | |
| # * before_test_script : name of script to run before tests, if any. | |
| # Will be executed outside container, from | |
| # workspace root. Can be used to start docker | |
| # containers as services for testing. | |
| # * test_script : name of custom test script, if any. Will be executed | |
| # inside container, from build subdirectory. | |
| # * travis_branch : value of TRAVIS_BRANCH environment variable, | |
| # used for test skipping | |
| include: | |
| - name: Alpine, gcc | |
| id: alpine | |
| container: alpine | |
| build_script: build.sh | |
| os: ubuntu-22.04 | |
| - name: Alpine, clang 32-bit | |
| id: alpine_32bit | |
| container: alpine_32bit | |
| build_script: build.sh | |
| test_script: test.sh | |
| travis_branch: alpine_32bit | |
| os: ubuntu-22.04 | |
| - name: Fedora Rawhide, clang++ | |
| id: fedora_rawhide | |
| travis_branch: fedora_rawhide | |
| container: fedora_rawhide | |
| build_script: build.sh | |
| os: ubuntu-22.04 | |
| - name: Ubuntu 24.04, gcc | |
| id: ubuntu_24.04 | |
| travis_branch: ubuntu_2404 | |
| container: ubuntu_24.04 | |
| before_test_script: services.sh | |
| build_script: build.sh | |
| test_script: test.sh | |
| os: ubuntu-22.04 | |
| - name: Ubuntu 22.04, gcc | |
| id: ubuntu_22.04 | |
| travis_branch: ubuntu_2204 | |
| container: ubuntu_22.04 | |
| before_test_script: services.sh | |
| build_script: build.sh | |
| test_script: test.sh | |
| os: ubuntu-22.04 | |
| - name: Ubuntu 22.04, clang ASAN | |
| id: asan | |
| travis_branch: sanitize | |
| container: ubuntu_22.04 | |
| build_script: build.sh | |
| test_script: test.sh | |
| os: ubuntu-22.04 | |
| - name: Ubuntu 20.04, gcc | |
| id: ubuntu_20.04 | |
| travis_branch: ubuntu_2004 | |
| container: ubuntu_20.04 | |
| use_avx2: true | |
| build_script: build.sh | |
| test_script: test.sh | |
| os: ubuntu-22.04 | |
| - name: Ubuntu 20.04, coverage | |
| id: coverage | |
| travis_branch: ubuntu_2004 | |
| container: ubuntu_20.04 | |
| before_test_script: services.sh | |
| build_script: build.sh | |
| test_script: test.sh | |
| os: ubuntu-22.04 | |
| - name: Ubuntu 20.04, benchmarks | |
| id: benchmarks | |
| travis_branch: ubuntu_2004 | |
| container: ubuntu_20.04 | |
| build_script: build.sh | |
| test_script: test.sh | |
| os: ubuntu-22.04 | |
| - name: Ubuntu 20.04, Intel compiler | |
| id: icc | |
| container: icc | |
| build_script: build.sh | |
| os: ubuntu-22.04 | |
| name: ${{ matrix.name }} | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| # bash is needed to use ${CONTAINER_REGISTRY_USER,,}, which forces the | |
| # username to lower-case as required by docker. | |
| shell: bash | |
| steps: | |
| - name: Set variables | |
| # This logic needs to be kept in sync between linux_build.yml and converity_scan.yml | |
| run: | | |
| CONTAINER_TAG_CLEAN=$(echo ${CONTAINER_TAG} | tr -d -c "[:alnum:].-") | |
| echo "CONTAINER_TAG_CLEAN=${CONTAINER_TAG_CLEAN}" | |
| echo "CONTAINER_TAG_CLEAN=${CONTAINER_TAG_CLEAN}" >> ${GITHUB_ENV} | |
| CACHE_CONTAINER_TAG_CLEAN=$(echo ${CACHE_CONTAINER_TAG} | tr -d -c "[:alnum:].-") | |
| echo "CACHE_CONTAINER_TAG_CLEAN=${CACHE_CONTAINER_TAG_CLEAN}" | |
| echo "CACHE_CONTAINER_TAG_CLEAN=${CACHE_CONTAINER_TAG_CLEAN}" >> ${GITHUB_ENV} | |
| echo "CONTAINER_NAME_FULL=${CONTAINER_REGISTRY}/${CONTAINER_REGISTRY_USER,,}/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN}" >>${GITHUB_ENV} | |
| # Work around segfaults in ASan/MSan jobs | |
| # Cf https://github.com/libjpeg-turbo/libjpeg-turbo/commit/2dfe6c0fe9e18671105e94f7cbf044d4a1d157e6 | |
| # and https://github.com/actions/runner-images/issues/9491 | |
| - name: Set up build | |
| run: | | |
| sudo sysctl vm.mmap_rnd_bits=28 | |
| - name: Checkout | |
| uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 | |
| - name: Login to Docker Hub | |
| if: env.CONTAINER_REGISTRY == 'docker.io' | |
| uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Login to GHCR | |
| if: env.CONTAINER_REGISTRY == 'ghcr.io' | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin | |
| # Pull build environment in forks or pull requests, unless [skip cache] is included in the commit message | |
| - name: Pull build environment | |
| if: "(github.repository_owner != 'OSGeo' || github.event_name == 'pull_request') && !contains(github.event.head_commit.message, '[skip cache]')" | |
| run: | | |
| docker pull ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN} || true | |
| docker pull ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CACHE_CONTAINER_TAG_CLEAN} || true | |
| docker pull ${CONTAINER_NAME_FULL} || true | |
| echo "DOCKER_BUILD_CACHE_FROM=--cache-from ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN} --cache-from ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CACHE_CONTAINER_TAG_CLEAN} --cache-from ${CONTAINER_NAME_FULL}" >>${GITHUB_ENV} | |
| - name: Prepare build context | |
| run: | | |
| mkdir docker-build-context | |
| cp autotest/requirements.txt docker-build-context | |
| - name: Update build environment | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| run: | | |
| # FIXME: for some reason, the fedora rawhide container pushed by | |
| # master job is corrupted (looks like it contains an outdated layer | |
| # symlinking libssl.so.3 to an older version of the actual file), | |
| # once it is pushed. But in the job that generates it, | |
| # compilation & tests work fine. It looks like some weird caching | |
| # issue | |
| if test "${{ matrix.container }}" = "fedora_rawhide"; then | |
| DOCKER_BUILD_CACHE_FROM="" | |
| else | |
| BUILD_ARG_INLINE_CACHE="--build-arg BUILDKIT_INLINE_CACHE=1" | |
| fi | |
| docker build \ | |
| ${BUILD_ARG_INLINE_CACHE} \ | |
| ${DOCKER_BUILD_CACHE_FROM} \ | |
| -t ${CONTAINER_NAME_FULL} \ | |
| -f .github/workflows/${{ matrix.container }}/Dockerfile.ci \ | |
| docker-build-context | |
| # Get the architecture so we can use it as part of the cache key, | |
| # but only if we are going to use avx2 in the build. If we are not, | |
| # including the arch will cause unnecessary cache misses. | |
| - name: Get Architecture | |
| id: get-arch | |
| if: matrix.use_avx2 | |
| run: | | |
| export ARCH=$(cc -march=native -### -E - < /dev/null 2>&1 | sed -ne 's/.*cc1 .*-march=\([^ "]*\)[ "].*/\1/p') | |
| echo "Architecture: $ARCH" | |
| echo "arch=$ARCH" >> $GITHUB_OUTPUT | |
| # cache the .ccache directory | |
| # key it on the runner os, build type, deps, and arch | |
| # It's especially important to include arch in the key because we | |
| # may get runtime errors with -mavx2 from objects built on a | |
| # different architecture. | |
| - name: Restore build cache | |
| id: restore-cache | |
| uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 | |
| with: | |
| path: ${{ github.workspace }}/.ccache | |
| key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }} | |
| restore-keys: | | |
| ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }} | |
| ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }} | |
| - name: Prepare ccache | |
| run: | | |
| mkdir -p ${{ github.workspace }}/.ccache | |
| chmod -R a+rw ${{ github.workspace }}/.ccache | |
| docker run --rm \ | |
| -v ${{ github.workspace }}/.ccache:/.ccache \ | |
| -u $(id -u ${USER}):$(id -g ${USER}) \ | |
| ${CONTAINER_NAME_FULL} \ | |
| sh -c "ccache -M 1G && ccache -sp && ccache -z" | |
| # FIXME the default BUILD_CMD here isn't working...we get an error | |
| # about the quotes not matching. | |
| - name: Build | |
| env: | |
| TRAVIS: yes | |
| TRAVIS_BRANCH: ${{ matrix.travis_branch }} | |
| BUILD_NAME: ${{ matrix.travis_branch }} | |
| run: | | |
| if test -f ".github/workflows/${{ matrix.id }}/${{ matrix.build_script }}"; then | |
| BUILD_CMD="$(pwd)/.github/workflows/${{ matrix.id }}/${{ matrix.build_script }}" | |
| else | |
| BUILD_CMD="sh -c 'cmake .. && make -j$(nproc)'" | |
| fi | |
| # For cache | |
| mkdir -p .gdal | |
| mkdir -p build-${{ matrix.id }} | |
| docker run --name gdal-build \ | |
| --rm \ | |
| -e CI \ | |
| -e GITHUB_WORKFLOW \ | |
| -e TRAVIS \ | |
| -e TRAVIS_BRANCH \ | |
| -e BUILD_NAME \ | |
| -e "GDAL_SOURCE_DIR=$(pwd)" \ | |
| -u $(id -u ${USER}):$(id -g ${USER}) \ | |
| -v $(pwd)/.gdal:/.gdal:rw \ | |
| -v $(pwd):$(pwd):rw \ | |
| -v ${{ github.workspace }}/.ccache:/.ccache:rw \ | |
| --workdir $(pwd)/build-${{ matrix.id }} \ | |
| ${CONTAINER_NAME_FULL} \ | |
| ${BUILD_CMD} | |
| - name: Summarize ccache | |
| run: | | |
| docker run --rm \ | |
| -v ${{ github.workspace }}/.ccache:/.ccache \ | |
| -u $(id -u ${USER}):$(id -g ${USER}) \ | |
| ${CONTAINER_NAME_FULL} \ | |
| ccache -s | |
| - name: Save build cache | |
| uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 | |
| with: | |
| path: ${{ github.workspace }}/.ccache | |
| key: ${{ steps.restore-cache.outputs.cache-primary-key }} | |
| - name: Start test services | |
| if: matrix.before_test_script | |
| run: | | |
| .github/workflows/${{ matrix.id }}/${{ matrix.before_test_script }} | |
| # --security-opt seccomp=unconfined, so that the userfaulfd syscall is availabledocker run \ | |
| - name: Run tests | |
| env: | |
| TRAVIS: yes | |
| TRAVIS_BRANCH: ${{ matrix.travis_branch }} | |
| BUILD_NAME: ${{ matrix.travis_branch }} | |
| run: | | |
| if test -f ".github/workflows/${{ matrix.id }}/${{ matrix.test_script }}"; then | |
| TEST_CMD="$(pwd)/.github/workflows/${{ matrix.id }}/${{ matrix.test_script }}" | |
| else | |
| TEST_CMD="ctest -V -j $(nproc)" | |
| fi | |
| if test "${{ matrix.id }}" = "benchmarks"; then | |
| if test -f /sys/devices/system/cpu/intel_pstate/no_turbo; then | |
| echo "Disable TurboBoost" | |
| echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo | |
| fi | |
| fi | |
| # For cache | |
| mkdir -p .gdal | |
| docker run \ | |
| -e CI \ | |
| -e GITHUB_WORKFLOW \ | |
| -e TRAVIS \ | |
| -e TRAVIS_BRANCH \ | |
| -e BUILD_NAME \ | |
| -e "GDAL_SOURCE_DIR=$(pwd)" \ | |
| -u $(id -u ${USER}):$(id -g ${USER}) \ | |
| --security-opt seccomp=unconfined \ | |
| --add-host=host.docker.internal:host-gateway \ | |
| --rm \ | |
| -v $(pwd)/.gdal:/.gdal \ | |
| -v $(pwd):$(pwd) \ | |
| --workdir $(pwd)/build-${{ matrix.id }} \ | |
| ${CONTAINER_NAME_FULL} \ | |
| ${TEST_CMD} | |
| - name: Coveralls | |
| uses: coverallsapp/github-action@cfd0633edbd2411b532b808ba7a8b5e04f76d2c8 # v2.3.4 | |
| if: ${{ matrix.id == 'coverage' }} | |
| with: | |
| format: lcov | |
| file: build-coverage/gdal_filtered.info | |
| - name: Push build environment | |
| if: github.event_name == 'push' | |
| continue-on-error: true | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| run: | | |
| docker push ${CONTAINER_NAME_FULL} | |
| - name: Upload coverage artifacts | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| if: ${{ matrix.id == 'coverage' }} | |
| with: | |
| name: coverage_index.html | |
| path: build-${{ matrix.id }}/coverage_html/index.html | |
| - name: Upload coverage artifacts | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| if: ${{ matrix.id == 'coverage' }} | |
| with: | |
| name: HTML | |
| path: build-${{ matrix.id }}/coverage_html/* | |
| - name: Deploy ssh key (for coverage) | |
| if: ${{ matrix.id == 'coverage' && github.event_name == 'push' && github.ref_name == 'master' && github.repository == 'OSGeo/gdal' }} | |
| shell: bash -l {0} | |
| run: | | |
| mkdir $HOME/.ssh && echo "${{ secrets.GDAL_TEST_COVERAGE_RESULTS_SSH_KEY }}" > $HOME/.ssh/id_rsa | |
| chmod 700 $HOME/.ssh && chmod 600 $HOME/.ssh/id_rsa | |
| ssh-keyscan -t rsa github.com >> $HOME/.ssh/known_hosts | |
| eval `ssh-agent -s` | |
| ssh-add $HOME/.ssh/id_rsa | |
| - name: Deploy to https://github.com/OSGeo/gdal-test-coverage-results | |
| if: ${{ matrix.id == 'coverage' && github.event_name == 'push' && github.ref_name == 'master' && github.repository == 'OSGeo/gdal' }} | |
| shell: bash -l {0} | |
| run: | | |
| set -x | |
| set -e | |
| mkdir -p output_html/coverage_html | |
| cp -r $GITHUB_WORKSPACE/build-${{ matrix.id }}/coverage_html/* output_html/coverage_html/ | |
| cd output_html | |
| git init | |
| git config user.email "[email protected]" | |
| git config user.name "GDAL test coverage results bot" | |
| git remote add origin [email protected]:gdalautotest-coverage-results/gdalautotest-coverage-results.github.io | |
| git remote -v | |
| echo "Results of coverage of GDAL autotest See https://gdalautotest-coverage-results.github.io/coverage_html/index.html" > README.md | |
| git add -A | |
| git commit -m "Update with OSGeo/gdal commit $GITHUB_SHA" | |
| git push -f origin master |