Skip to content

Implement RFC 104: Adding a "gdal" front-end command line interface #6966

Implement RFC 104: Adding a "gdal" front-end command line interface

Implement RFC 104: Adding a "gdal" front-end command line interface #6966

Workflow file for this run

# build GDAL library and to run test with CMake
name: Build and test with CMake
on:
push:
paths-ignore:
- 'doc/**'
- 'docker/**'
branches-ignore:
- 'backport**'
- 'dependabot**'
pull_request:
paths-ignore:
- 'doc/**'
- 'docker/**'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
env:
CMAKE_UNITY_BUILD: OFF
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
build-linux-ubuntu-focal:
runs-on: ubuntu-20.04
env:
CMAKE_OPTIONS: -DPython_LOOKUP_VERSION=3.8 -DUSE_CCACHE=ON -DSWIG_REGENERATE_PYTHON=ON
cache-name: cmake-ubuntu-focal
steps:
- name: Checkout GDAL
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- name: Setup cache
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
id: cache
with:
path: ${{ github.workspace }}/.ccache
key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.base_ref }}${{ github.ref_name }}-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-${{ env.cache-name }}-${{ github.base_ref }}
${{ runner.os }}-${{ env.cache-name }}
- name: Install dependency
run: |
sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
sudo apt-get update
sudo apt-get install -y -q bison libaec-dev libjpeg-dev libgif-dev liblzma-dev libzstd-dev libgeos-dev git \
libcurl4-gnutls-dev libproj-dev libxml2-dev libxerces-c-dev libnetcdf-dev netcdf-bin \
libpoppler-dev libpoppler-private-dev gpsbabel libhdf4-alt-dev libhdf5-serial-dev libpodofo-dev poppler-utils \
libfreexl-dev unixodbc-dev libwebp-dev libepsilon-dev liblcms2-2 libcrypto++-dev libkml-dev \
libmysqlclient-dev libarmadillo-dev wget libfyba-dev libjsoncpp-dev libexpat1-dev \
libclc-dev ocl-icd-opencl-dev libsqlite3-dev sqlite3-pcre libpcre3-dev libspatialite-dev libsfcgal-dev fossil libcairo2-dev libjson-c-dev libdeflate-dev liblz4-dev libblosc-dev libarchive-dev \
libqhull-dev libcfitsio-dev libogdi-dev libopenjp2-7-dev libheif-dev \
python3-dev libpython3-dev libpython3.8-dev python3.8-dev python3-numpy python3-lxml pyflakes python3-setuptools python3-pip python3-venv \
python3-pytest swig doxygen texlive-latex-base make cppcheck ccache g++ \
libpq-dev libpqtypes-dev postgresql-12 postgresql-12-postgis-3 postgresql-client-12 postgresql-12-postgis-3-scripts
# MSSQL: client side
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
sudo apt-get update
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev
# HANA: client side
# Install hdbsql tool
curl -v -j -k -s -L -H "Cookie: eula_3_2_agreed=tools.hana.ondemand.com/developer-license-3_2.txt" https://tools.hana.ondemand.com/additional/hanaclient-latest-linux-x64.tar.gz --output hanaclient-latest-linux-x64.tar.gz \
&& tar -xvf hanaclient-latest-linux-x64.tar.gz \
&& sudo mkdir /usr/sap \
&& sudo ./client/hdbinst -a client --sapmnt=/usr/sap \
&& rm -rf client \
&& rm hanaclient*
export PATH=/usr/sap/hdbclient:$PATH
# Download and compile odbc-cpp-wrapper
sudo apt-get install -y -q cmake
wget https://github.com/SAP/odbc-cpp-wrapper/archive/refs/tags/v1.1.tar.gz -O odbc-cpp-wrapper.tar.gz \
&& mkdir odbc-cpp-wrapper \
&& tar -xvf odbc-cpp-wrapper.tar.gz -C odbc-cpp-wrapper --strip-components=1 \
&& mkdir odbc-cpp-wrapper/build \
&& cd odbc-cpp-wrapper/build \
&& /usr/bin/cmake .. \
&& make -j 2 \
&& sudo make install \
&& cd ../.. \
&& rm -rf odbc-cpp-wrapper
sudo ldconfig
#
# Install Arrow C++
sudo apt-get install -y -V ca-certificates lsb-release wget
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt-get update
sudo apt-get install -y -V libarrow-dev libparquet-dev libarrow-dataset-dev
#
# Workaround bug in ogdi packaging
sudo ln -s /usr/lib/ogdi/libvrf.so /usr/lib
#
python3 -m pip install -U pip wheel setuptools numpy importlib_metadata
python3 -m pip install -r $GITHUB_WORKSPACE/autotest/requirements.txt
- name: Build libjxl
run: |
# Build libjxl
# libjxl being still unstable, if the main branch fails to compile/test
# you can replace JXL_TREEISH=main by JXL_TREEISH=sha1_of_known_working_commit
JXL_TREEISH=main
git clone https://github.com/libjxl/libjxl.git --recursive \
&& cd libjxl \
&& git checkout ${JXL_TREEISH} \
&& mkdir build \
&& cd build \
&& sudo apt-get install -y --no-install-recommends libgflags-dev libbrotli-dev \
&& /usr/bin/cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DJPEGXL_FORCE_SYSTEM_BROTLI:BOOL=ON -DJPEGXL_ENABLE_JPEGLI_LIBJPEG=OFF -DCMAKE_INSTALL_PREFIX=/usr .. \
&& make -j$(nproc) \
&& sudo make -j$(nproc) install \
&& cd ../.. \
&& rm -rf libjxl
- name: Build libQB3
run: |
# Build libQB3
# Used by the MRF driver
git clone https://github.com/lucianpls/QB3.git \
&& mkdir QB3/QB3lib/build \
&& cd QB3/QB3lib/build \
&& /usr/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .. \
&& make -j$(nproc) \
&& sudo make -j$(nproc) install \
&& cd ../../.. \
&& rm -rf QB3
- name: Build libbasisu
run: |
# Used by the BASISU and KTX2 drivers
git clone -b cmake https://github.com/rouault/basis_universal.git \
&& mkdir basis_universal/build \
&& cd basis_universal/build \
&& /usr/bin/cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .. \
&& make -j$(nproc) \
&& sudo make -j$(nproc) install \
&& cd ../..\
&& rm -rf basis_universal
- name: Install pdfium
run: |
wget -q https://github.com/rouault/pdfium_build_gdal_3_10/releases/download/pdfium_6677_v1/install-ubuntu2004-rev6677.tar.gz \
&& tar -xzf install-ubuntu2004-rev6677.tar.gz \
&& sudo chown -R root:root install \
&& sudo mv install/lib/* /usr/lib/ \
&& sudo mv install/include/* /usr/include/ \
&& sudo rm -rf install-ubuntu2004-rev6677.tar.gz install \
&& sudo apt-get update -y \
&& sudo apt-get install -y --fix-missing --no-install-recommends liblcms2-dev
- name: Configure ccache
run: |
echo CCACHE_BASEDIR=$PWD >> ${GITHUB_ENV}
echo CCACHE_DIR=$PWD/.ccache >> ${GITHUB_ENV}
echo CCACHE_MAXSIZE=250M >> ${GITHUB_ENV}
ccache -z
working-directory: ${{ github.workspace }}
- name: Configure
run: |
export PATH=$CMAKE_DIR:/usr/local/bin:/usr/bin:/bin # Avoid CMake config from brew etc.
mkdir -p $GITHUB_WORKSPACE/superbuild
cd $GITHUB_WORKSPACE/superbuild
printf "cmake_minimum_required(VERSION 3.16)\nproject(test)\nenable_testing()\nadd_subdirectory(gdal)" > CMakeLists.txt
ln -s .. gdal
mkdir build
cd build
cmake ${CMAKE_OPTIONS} -Werror=dev -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror -DGDAL_USE_PUBLICDECOMPWT:BOOL=ON -DPUBLICDECOMPWT_URL=https://github.com/rouault/PublicDecompWT .. -DWERROR_DEV_FLAG="-Werror=dev"
- name: Build
run: |
cmake --build $GITHUB_WORKSPACE/superbuild/build -- -j$(nproc)
env:
GIT_LFS_SKIP_SMUDGE: 1 # for PublicDecompWT github repository clone
- name: Build fake ossfuzz fuzzers
run: |
cd $GITHUB_WORKSPACE/superbuild/build
g++ -c $GITHUB_WORKSPACE/fuzzers/fuzzingengine.cpp
ar r libFuzzingEngine.a fuzzingengine.o
CXX=g++ CXXFLAGS="${CXXFLAGS} -I$PWD/gdal/port -I$PWD/gdal/gcore -L$PWD" LIBGDAL="-L$PWD/gdal -lgdal" SRC=/tmp OUT=/tmp $GITHUB_WORKSPACE/fuzzers/build_google_oss_fuzzers.sh
OUT=/tmp $GITHUB_WORKSPACE/fuzzers/build_seed_corpus.sh
- name: test autotest/cpp/gdallimits
run: |
sudo apt-get install -y numactl
#
echo "Limits without restrictions:"
LD_LIBRARY_PATH=$GITHUB_WORKSPACE/superbuild/build $GITHUB_WORKSPACE/superbuild/build/gdal/autotest/cpp/gdallimits
#
echo "Limits with ulimit -m 100000; numactl -C 0"
(ulimit -m 100000; LD_LIBRARY_PATH=$GITHUB_WORKSPACE/superbuild/build numactl -C 0 $GITHUB_WORKSPACE/superbuild/build/gdal/autotest/cpp/gdallimits) > out.txt
cat out.txt
grep "CPLGetNumCPUs = 1" out.txt >/dev/null
grep "CPLGetUsablePhysicalRAM = 97 MB" out.txt >/dev/null
- name: test (with command targets)
run: |
cmake --build $GITHUB_WORKSPACE/superbuild/build --target quicktest -- -j$(nproc)
- name: test (with ctest)
run: |
cd $GITHUB_WORKSPACE/superbuild/build
ctest -V
- name: install and uninstall
run: |
cmake --build $GITHUB_WORKSPACE/superbuild/build --target install -- -j$(nproc)
cmake --build $GITHUB_WORKSPACE/superbuild/build --target uninstall
find $GITHUB_WORKSPACE/install-gdal || /bin/true
- name: install
run: |
export PATH=$CMAKE_DIR:/usr/local/bin:/usr/bin:/bin # Avoid CMake config from brew etc.
touch $GITHUB_WORKSPACE/man/man1/gdalinfo.1
touch $GITHUB_WORKSPACE/man/man1/gdaladdo.1
(cd $GITHUB_WORKSPACE/superbuild/build; cmake ..)
cmake --build $GITHUB_WORKSPACE/superbuild/build --target install -- -j$(nproc)
test -f $GITHUB_WORKSPACE/install-gdal/share/man/man1/gdalinfo.1
test -f $GITHUB_WORKSPACE/install-gdal/share/man/man1/gdaladdo.1
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-gdal/lib
$GITHUB_WORKSPACE/install-gdal/bin/gdalinfo --version
PYTHONPATH=$GITHUB_WORKSPACE/install-gdal/lib/python3.8/site-packages python3 -c "from osgeo import gdal;print(gdal.VersionInfo(None))"
# Test fix for https://github.com/conda-forge/gdal-feedstock/issues/995
PYTHONWARNINGS="error" PYTHONPATH=$GITHUB_WORKSPACE/install-gdal/lib/python3.8/site-packages python3 -c "from osgeo import gdal"
PYTHONPATH=$GITHUB_WORKSPACE/install-gdal/lib/python3.8/site-packages python3 $GITHUB_WORKSPACE/scripts/check_doc.py
- name: CMake with rpath
run: |
export PATH=$CMAKE_DIR:/usr/local/bin:/usr/bin:/bin # Avoid CMake config from brew etc.
(cd $GITHUB_WORKSPACE/superbuild/build; cmake .. "-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal-with-rpath" "-DCMAKE_INSTALL_RPATH=$GITHUB_WORKSPACE/install-gdal-with-rpath/lib")
cmake --build $GITHUB_WORKSPACE/superbuild/build --target install -- -j$(nproc)
# For some reason, during the install phase of above invocation, the Python bindings are rebuilt after the build phase, and without the rpath... Can't reproduce that locally
# PYTHONPATH=$GITHUB_WORKSPACE/install-gdal-with-rpath/lib/python3.8/site-packages python -c "from osgeo import gdal;print(gdal.VersionInfo(None))"
- name: Rerun using Mono
run: |
export PATH=$CMAKE_DIR:/usr/local/bin:/usr/bin:/bin # Avoid CMake config from brew etc.
cd $GITHUB_WORKSPACE/superbuild/build
rm -rf swig/csharp
cmake ${CMAKE_OPTIONS} -DCSHARP_MONO=ON -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal -UCMAKE_INSTALL_RPATH -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror ..
cmake --build $GITHUB_WORKSPACE/superbuild/build --target install -- -j$(nproc)
# Below fails with errors like 'System.InvalidProgramException: Invalid IL code in CreateData:Main (string[]): IL_00c4: callvirt 0x0a00000c'
# ctest -V -R "^csharp.*"
- name: Standalone Python bindings build from source
run: |
(cd $GITHUB_WORKSPACE/superbuild/build/gdal/swig/python && python setup.py sdist)
mv $GITHUB_WORKSPACE/superbuild/build/gdal/swig/python/dist/gdal-*.tar.gz gdal-python.tar.gz
mv $GITHUB_WORKSPACE/superbuild/build/gdal/swig/python gdal-swig-python
python3 -m venv myvenv
source myvenv/bin/activate
# Set PATH so that gdal-config is found
PATH=$GITHUB_WORKSPACE/install-gdal/bin:$PATH pip install gdal-python.tar.gz[numpy]
LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-gdal/lib python -c "from osgeo import gdal_array"
which gdal_edit
cp $GITHUB_WORKSPACE/autotest/gcore/data/byte.tif .
LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-gdal/lib ldd myvenv/lib/python3.8/site-packages/osgeo/_gdal.cpython-38-x86_64-linux-gnu.so
LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-gdal/lib gdal_edit byte.tif -mo FOO=BAR
rm -f myvenv/bin/gdal_edit
rm -f myvenv/bin/gdal_edit.py
- name: Standalone gdal-utils package from wheel
run: |
mv gdal-swig-python $GITHUB_WORKSPACE/superbuild/build/gdal/swig/python
(cd $GITHUB_WORKSPACE/superbuild/build/gdal/swig/python/gdal-utils && python setup.py bdist_wheel)
mv $GITHUB_WORKSPACE/superbuild/build/gdal/swig/python/gdal-utils/dist/*.whl gdal_utils-X.Y.Z.T-py3-none-any.whl
rm -rf $GITHUB_WORKSPACE/superbuild/build/gdal/swig/python
python3 -m venv myvenv
source myvenv/bin/activate
# Set PATH so that gdal-config is found
PATH=$GITHUB_WORKSPACE/install-gdal/bin:$PATH pip install gdal_utils-X.Y.Z.T-py3-none-any.whl
which gdal_edit
cp $GITHUB_WORKSPACE/autotest/gcore/data/byte.tif .
LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install-gdal/lib gdal_edit byte.tif -mo FOO=BAR
pip uninstall -y gdal-utils
- name: Test post-install usage (with pkg-config)
run: |
./autotest/postinstall/test_pkg-config.sh $GITHUB_WORKSPACE/install-gdal
- name: Test post-install usage (with gdal-config)
run: |
./autotest/postinstall/test_gdal-config.sh $GITHUB_WORKSPACE/install-gdal
- name: Test post-install usage (with CMake)
run: |
./autotest/postinstall/test_cmake.sh $GITHUB_WORKSPACE/install-gdal
- name: Rerun with static linkage
run: |
export PATH=$CMAKE_DIR:/usr/local/bin:/usr/bin:/bin # Avoid CMake config from brew etc.
cd $GITHUB_WORKSPACE/superbuild/build
rm -f CMakeCache.txt
cmake ${CMAKE_OPTIONS} .. \
-DBUILD_SHARED_LIBS=OFF \
-DBUILD_TESTING=OFF \
-DCSHARP_MONO=OFF \
-DGDAL_USE_HDF4=OFF \
-DGDAL_USE_HDF5=OFF \
-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal-static
cmake --build $GITHUB_WORKSPACE/superbuild/build --target install -- -j$(nproc)
- name: Test post-install usage (static, with pkg-config)
run: |
./autotest/postinstall/test_pkg-config.sh $GITHUB_WORKSPACE/install-gdal-static --static
- name: Test post-install usage (static, with gdal-config)
run: |
./autotest/postinstall/test_gdal-config.sh $GITHUB_WORKSPACE/install-gdal-static --static
- name: Test post-install usage (static, with CMake)
run: |
./autotest/postinstall/test_cmake.sh $GITHUB_WORKSPACE/install-gdal-static --static
- name: ccache statistics
run: ccache -s
build-windows-msys2-mingw:
runs-on: windows-2022
env:
generator: MinGW Makefiles
cache-name: cmake-mingw64
defaults:
run:
shell: msys2 {0}
steps:
# To avoid git clone to mess with the line endings of GDAL autotest data
# files that look like text, but should be handled as binary content
- name: Set git core.autocrlf to false
shell: bash -l {0}
run: |
git config --global core.autocrlf false
- name: Checkout GDAL
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- name: Install development packages
uses: msys2/setup-msys2@ddf331adaebd714795f1042345e6ca57bd66cea8 # v2.24.1
with:
msystem: MINGW64
update: true
# Disable mingw-w64-x86_64-crypto++ as it crashes at runtime on eedai_3 test. Likely a build issue with crypto++ itself
install: |
base-devel git mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache
mingw-w64-x86_64-pcre mingw-w64-x86_64-xerces-c mingw-w64-x86_64-zstd mingw-w64-x86_64-libarchive
mingw-w64-x86_64-geos mingw-w64-x86_64-libspatialite mingw-w64-x86_64-proj
mingw-w64-x86_64-cgal mingw-w64-x86_64-libfreexl mingw-w64-x86_64-hdf5 mingw-w64-x86_64-netcdf mingw-w64-x86_64-poppler mingw-w64-x86_64-podofo mingw-w64-x86_64-postgresql
mingw-w64-x86_64-libgeotiff mingw-w64-x86_64-libpng mingw-w64-x86_64-libtiff mingw-w64-x86_64-openjpeg2
mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-numpy mingw-w64-x86_64-python-pytest mingw-w64-x86_64-python-setuptools mingw-w64-x86_64-python-lxml mingw-w64-x86_64-swig mingw-w64-x86_64-python-psutil mingw-w64-x86_64-blosc mingw-w64-x86_64-libavif
- name: Setup cache
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
id: cache
with:
path: ${{ github.workspace }}\.ccache
key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.base_ref }}${{ github.ref_name }}-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-${{ env.cache-name }}-${{ github.base_ref }}
${{ runner.os }}-${{ env.cache-name }}
- name: Configure ccache
run: |
echo CCACHE_BASEDIR=$PWD >> ${GITHUB_ENV}
echo CCACHE_DIR=$PWD/.ccache >> ${GITHUB_ENV}
echo CCACHE_MAXSIZE=250M >> ${GITHUB_ENV}
ccache -z
working-directory: ${{ github.workspace }}
- name: populate JAVA_HOME
run: |
echo "JAVA_HOME=$JAVA_HOME_11_X64" >> ${GITHUB_ENV}
- name: Install python dependencies for autotest
run: |
# One of the dependencies of jsonschema 4.18 is rpds_py which requires a Rust compiler
python -m pip install "jsonschema<4.18"
python -m pip install -r autotest/requirements.txt
# Disable mySQL since C:/mysql/lib/mysqlclient.lib (unrelated to msys) is found, which causes linking issues
# Set explicitly CMAKE_C|CXX_COMPILER otherwise C:/ProgramData/chocolatey/bin/gcc.exe would be used
# Disable GDAL_ENABLE_DRIVER_HDF5 because of https://github.com/OSGeo/gdal/issues/11181
- name: Configure
run: |
cmake -S . -B build -G "${generator}" -Werror=dev \
-DCMAKE_BUILD_TYPE=release \
-DCMAKE_C_FLAGS=-Werror \
-DCMAKE_CXX_FLAGS=-Werror \
-DWERROR_DEV_FLAG=-Werror=dev \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_PREFIX_PATH=/mingw64 \
-DCMAKE_INSTALL_PREFIX=$PWD/install-gdal \
"-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD}" \
-DGDAL_USE_MYSQL:BOOL=OFF \
-DGDAL_ENABLE_DRIVER_HDF5=OFF
working-directory: ${{ github.workspace }}
- name: Build
run: cmake --build build -j 3
working-directory: ${{ github.workspace }}
- name: test
run: cmake --build build --target quicktest
working-directory: ${{ github.workspace }}
- name: test (with ctest)
run: |
ctest --test-dir $GITHUB_WORKSPACE/build -C Release -V -j 3
env:
BUILD_NAME: mingw64
SKIP_OGR_GMLAS_HUGE_PROCESSING_TIME: YES
SKIP_OGR_GMLAS_HTTP_RELATED: YES
SKIP_TIFF_JPEG12: YES
SKIP_VSIRAR: YES
working-directory: ${{ github.workspace }}
- name: Install
run: cmake --build build --target install -j 3
working-directory: ${{ github.workspace }}
- name: Test post-install usage (with pkg-config)
run: ./autotest/postinstall/test_pkg-config.sh $PWD/install-gdal
working-directory: ${{ github.workspace }}
- name: ccache statistics
run: ccache -s
build-windows-conda:
runs-on: windows-2022
env:
architecture: x64
#generator: Visual Studio 17 2022
generator: Ninja
cache-name: cmake-msvc
steps:
# To avoid git clone to mess with the line endings of GDAL autotest data
# files that look like text, but should be handled as binary content
- name: Set git core.autocrlf to false
run: |
git config --global core.autocrlf false
- name: Checkout GDAL
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
- name: populate JAVA_HOME
shell: pwsh
run: |
echo "JAVA_HOME=$env:JAVA_HOME_11_X64" >> %GITHUB_ENV%
- uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
with:
activate-environment: gdalenv
miniforge-version: latest
use-mamba: true
auto-update-conda: true
use-only-tar-bz2: false
- name: Install dependency
shell: bash -l {0}
run: |
conda install --yes --quiet curl libiconv icu python=3.10 swig numpy pytest pytest-env pytest-benchmark filelock zlib lxml jsonschema setuptools
# FIXME: remove libnetcdf=4.9.2=nompi_h5902ca5_107 pinning as soon as https://github.com/conda-forge/libnetcdf-feedstock/issues/182 is resolved
conda install --yes --quiet proj geos hdf4 hdf5 kealib \
libnetcdf=4.9.2=nompi_h5902ca5_107 openjpeg poppler libtiff libpng xerces-c expat libxml2 kealib json-c \
cfitsio freexl geotiff libjpeg-turbo libpq libspatialite libwebp-base pcre pcre2 postgresql \
sqlite tiledb zstd cryptopp cgal doxygen librttopo libkml openssl xz \
openjdk ant qhull armadillo blas blas-devel libblas libcblas liblapack liblapacke blosc libarchive \
arrow-cpp pyarrow libaec libheif libavif cmake fsspec
- name: Check CMake version
shell: bash -l {0}
run: |
cmake --version
- name: Install pdfium
shell: bash -l {0}
run: |
curl -LOs https://github.com/rouault/pdfium_build_gdal_3_10/releases/download/pdfium_6677_v1/install-win10-vs2019-x64-rev6677.zip
unzip install-win10-vs2019-x64-rev6677.zip
mv install install-pdfium
- name: Remove conflicting libraries
shell: bash -l {0}
run: |
rm -rf C:/Strawberry || /bin/true
rm -rf "C:/Program Files/OpenSSL/lib" || /bin/true
- name: Configure
shell: bash -l {0}
# Disable MySQL because of "error LNK2038: mismatch detected for '_MSC_VER': value '1800' doesn't match value '1900' in ogrmysqldatasource.obj" and other errors
# Note that the leading space in CMAKE_C/CXX_FLAGS=" /WX" is due to using Bash on Windows that would
# otherwise interpret /bla has a file relative to the Bash root directory and would replace it by a path like c:\Program Files\git\WX
# BUILD_JAVA_BINDINGS=OFF because we get "Error occurred during initialization of VM. Corrupted ZIP library: C:\Miniconda\envs\gdalenv\Library\bin\zip.dll" when running java. Not reproducible on a standard VM
# Build PDF driver as plugin due to the PDFium build including libopenjp2 symbols which would conflict with external libopenjp2
run: |
mkdir -p $GITHUB_WORKSPACE/build
cmake -G "${generator}" -Werror=dev "-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal" "-DUSE_CCACHE=ON" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DGDAL_ENABLE_PLUGINS:BOOL=ON -DGDAL_ENABLE_PLUGINS_NO_DEPS:BOOL=ON -DGDAL_USE_PUBLICDECOMPWT:BOOL=ON -DPUBLICDECOMPWT_URL=https://github.com/rouault/PublicDecompWT -DBUILD_JAVA_BINDINGS=OFF -DBUILD_CSHARP_BINDINGS=ON -DGDAL_USE_MYSQL:BOOL=OFF -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DWERROR_DEV_FLAG="-Werror=dev" -DCMAKE_BUILD_TYPE=Release -DPDFIUM_ROOT=$GITHUB_WORKSPACE/install-pdfium -DGDAL_ENABLE_DRIVER_PDF_PLUGIN:BOOL=ON -DCMAKE_UNITY_BUILD=ON -DOGR_ENABLE_DRIVER_TAB_PLUGIN=OFF -DOGR_ENABLE_DRIVER_GEOJSON_PLUGIN=OFF
- name: Build
shell: bash -l {0}
run: cmake --build $GITHUB_WORKSPACE/build --config Release -j 2
env:
GIT_LFS_SKIP_SMUDGE: 1 # for PublicDecompWT github repository clone
- name: test
shell: bash -l {0}
run: |
cmake --build $GITHUB_WORKSPACE/build --config Release --target quicktest
- name: test (with ctest)
shell: bash -l {0}
run: |
ctest --test-dir $GITHUB_WORKSPACE/build -C Release -V -j 3
env:
SKIP_OGR_GMLAS_HUGE_PROCESSING_TIME: YES
SKIP_OGR_GMLAS_HTTP_RELATED: YES
SKIP_GDAL_HTTP_SSL_VERIFYSTATUS: YES
BUILD_NAME: "build-windows-conda"
- name: Install
shell: bash -l {0}
run: |
cmake --build $GITHUB_WORKSPACE/build --config Release --target install
export PATH=$GITHUB_WORKSPACE/install-gdal/bin:$PATH
gdalinfo --version
python -VV
PYTHONPATH=$GITHUB_WORKSPACE/install-gdal/lib/site-packages python -c "from osgeo import gdal;print(gdal.VersionInfo(None))"
export PATH=$GITHUB_WORKSPACE/install-gdal/Scripts:$PATH
PYTHONPATH=$GITHUB_WORKSPACE/install-gdal/lib/site-packages gdal_edit --version
- name: Check the build includes the expected drivers
shell: bash -l {0}
run: |
export PATH=$GITHUB_WORKSPACE/install-gdal/bin:$PATH
gdalinfo --formats > found_formats.txt
ogrinfo --formats >> found_formats.txt
cat found_formats.txt
cat $GITHUB_WORKSPACE/.github/workflows/windows_conda_expected_gdalinfo_formats.txt $GITHUB_WORKSPACE/.github/workflows/windows_conda_expected_ogrinfo_formats.txt > expected_formats.txt
dos2unix expected_formats.txt
dos2unix found_formats.txt
diff -u expected_formats.txt found_formats.txt
- name: Show gdal.pc
shell: bash -l {0}
run: cat $GITHUB_WORKSPACE/build/gdal.pc
- name: Test python setup.py install
shell: bash -l {0}
run: |
cd $GITHUB_WORKSPACE/build/swig/python
python setup.py install
build-windows-minimum:
runs-on: windows-2022
env:
architecture: x64
generator: Visual Studio 17 2022
GDAL_PYTHON_BINDINGS_WITHOUT_NUMPY: YES
steps:
# To avoid git clone to mess with the line endings of GDAL autotest data
# files that look like text, but should be handled as binary content
- name: Set git core.autocrlf to false
run: |
git config --global core.autocrlf false
- name: Checkout GDAL
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
with:
activate-environment: gdalenv
miniforge-version: latest
use-mamba: true
auto-update-conda: true
use-only-tar-bz2: false
- name: Install dependency
shell: bash -l {0}
run: |
conda install --yes --quiet proj pytest pytest-env pytest-benchmark filelock lxml cmake setuptools
- name: Check CMake version
shell: bash -l {0}
run: |
cmake --version
- name: Configure
shell: bash -l {0}
run: |
mkdir -p $GITHUB_WORKSPACE/build
cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DWERROR_DEV_FLAG="-Werror=dev"
- name: Build
shell: bash -l {0}
run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2
- name: Configure with even less dependencies
shell: bash -l {0}
run: |
rm -f build/CMakeCache.txt
cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DWERROR_DEV_FLAG="-Werror=dev"
- name: Configure with even less dependencies, and disabling all optional drivers
shell: bash -l {0}
run: |
rm -f build/CMakeCache.txt
cmake -A ${architecture} -G "${generator}" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -Werror=dev "-DCMAKE_CXX_COMPILER_LAUNCHER=clcache" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DGDAL_USE_EXTERNAL_LIBS:BOOL=OFF -DGDAL_USE_PNG_INTERNAL=OFF -DGDAL_USE_JPEG_INTERNAL=OFF -DGDAL_USE_JPEG12_INTERNAL=OFF -DGDAL_USE_GIF_INTERNAL=OFF -DGDAL_USE_LERC_INTERNAL=OFF -DGDAL_USE_LERCV1_INTERNAL=OFF -DGDAL_USE_QHULL_INTERNAL=OFF -DGDAL_USE_OPENCAD_INTERNAL=OFF -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DOGR_BUILD_OPTIONAL_DRIVERS=OFF -DGDAL_ENABLE_DRIVER_DERIVED=ON -DWERROR_DEV_FLAG="-Werror=dev"
- name: Build
shell: bash -l {0}
run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2
- name: Re-enable shapefile driver (otherwise lots of python tests would fail)
shell: bash -l {0}
run: |
cmake -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DOGR_ENABLE_DRIVER_SHAPE=ON -DOGR_ENABLE_DRIVER_SHAPE_PLUGIN=ON
- name: Build
shell: bash -l {0}
run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2
# Works around https://github.com/actions/runner-images/issues/10055
- name: Remove conflicting libraries
shell: bash -l {0}
run: |
find "C:/hostedtoolcache/windows/Java_Temurin-Hotspot_jdk" -name "msvcp140.dll" -exec rm {} \;
- name: test (with ctest)
shell: bash -l {0}
run: |
# gnm_test has suddenly started failing around June 16th 2024
# Related to image windows-latest 20240603.1.0 / actions/runner-images#10004
echo "def test_dummy(): pass" > $GITHUB_WORKSPACE/autotest/gnm/gnm_test.py
ctest --test-dir $GITHUB_WORKSPACE/build -C RelWithDebInfo -V -j 3
env:
SKIP_GDAL_HTTP_SSL_VERIFYSTATUS: YES
BUILD_NAME: "build-windows-minimum"
- name: Show gdal.pc
shell: bash -l {0}
run: cat $GITHUB_WORKSPACE/build/gdal.pc
build-mac:
# M1 MacOS runner: https://github.blog/changelog/2024-01-30-github-actions-introducing-the-new-m1-macos-runner-available-to-open-source/
runs-on: macOS-14
env:
# Disable Arrow/Parquet because the VM provides libraries in /usr/local/lib/
# that cause Illegal instruction error when running tests. I suspect the
# Arrow/Parquet libraries to be built with AVX2 support but the VM worker
# doesn't support it.
# Disable use of libkml since boost headers cannot be found
# Disable ODBC since the postinstall checks don't manage to find sql.h
CMAKE_OPTIONS: -DCFITSIO_ROOT=/opt/homebrew/Cellar/cfitsio -DPoppler_ROOT=/opt/homebrew/Cellar/poppler -DPROJ_ROOT=/opt/homebrew/Cellar/proj -DSPATIALITE_ROOT=/opt/homebrew/Cellar/libspatialite -DPostgreSQL_ROOT=/opt/homebrew/Cellar/libpq -DEXPAT_ROOT=/opt/homebrew/Cellar/expat -DXercesC_ROOT=/opt/homebrew/Cellar/xerces-c -DSQLite3_ROOT=/opt/homebrew/Cellar/sqlite -DOpenSSL_ROOT=/opt/homebrew/Cellar/openssl -DPNG_ROOT=/opt/homebrew/Cellar/libpng -DJPEG_ROOT=/opt/homebrew/Cellar/jpeg-turbo -DEXPECTED_JPEG_LIB_VERSION=80 -DOpenJPEG_ROOT=/opt/homebrew/Cellar/openjpeg -DCURL_ROOT=/opt/homebrew/Cellar/curl -DGDAL_USE_LIBKML=OFF -DGDAL_USE_ODBC=OFF
cache-name: cmake-macos
steps:
- name: Setup xcode
uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
with:
xcode-version: '15.4.0'
- name: Checkout GDAL
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- name: Setup cache
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
id: cache
with:
path: ${{ github.workspace }}/.ccache
key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.base_ref }}${{ github.ref_name }}-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-${{ env.cache-name }}-${{ github.base_ref }}
${{ runner.os }}-${{ env.cache-name }}
- run: |
cmake --version
- name: Install dependency
run: |
rm -rf /opt/homebrew/lib/node_modules # conflicts with node@18 from brew
brew update
brew outdated
# Attempt at fixing:
# => Pouring pkgconf--2.3.0_1.arm64_sonoma.bottle.tar.gz
# Error: The `brew link` step did not complete successfully
# The formula built, but is not symlinked into /opt/homebrew
# Could not symlink bin/pkg-config
# Target /opt/homebrew/bin/pkg-config
# is a symlink belonging to [email protected]. You can unlink it:
# brew unlink [email protected]
#
# To force the link and overwrite all conflicting files:
# brew link --overwrite pkgconf
#
# To list all files that would be deleted:
# brew link --overwrite pkgconf --dry-run
#
# Possible conflicting files are:
# /opt/homebrew/bin/pkg-config -> /opt/homebrew/Cellar/[email protected]/0.29.2_3/bin/pkg-config
# /opt/homebrew/share/aclocal/pkg.m4 -> /opt/homebrew/Cellar/[email protected]/0.29.2_3/share/aclocal/pkg.m4
# /opt/homebrew/share/man/man1/pkg-config.1 -> /opt/homebrew/Cellar/[email protected]/0.29.2_3/share/man/man1/pkg-config.1
rm -f /opt/homebrew/bin/pkg-config /opt/homebrew/share/aclocal/pkg.m4 /opt/homebrew/share/man/man1/pkg-config.1
brew install --overwrite [email protected] [email protected]
brew install postgresql || brew link postgresql
brew install pkg-config freexl libxml2 libspatialite geos proj libgeotiff openjpeg giflib libaec postgis poppler doxygen unixodbc jpeg-turbo aom jpeg-xl libheif libarchive libkml boost
brew install ccache swig
brew link --force sqlite
# gdal is automatically installed as a dependency for postgis
brew uninstall --ignore-dependencies gdal
# Workaround for https://github.com/Homebrew/homebrew-core/pull/170959#issuecomment-2379606442
sed -i .bak 's/hdf5_hl;hdf5;/hdf5_hl-shared;hdf5-shared;/g' "$(brew --prefix netcdf)/lib/cmake/netCDF/netCDFTargets.cmake"
- name: Configure ccache
run: |
echo CCACHE_BASEDIR=$PWD >> ${GITHUB_ENV}
echo CCACHE_DIR=$PWD/.ccache >> ${GITHUB_ENV}
echo CCACHE_MAXSIZE=250M >> ${GITHUB_ENV}
ccache -z
working-directory: ${{ github.workspace }}
- name: Configure
run: |
mkdir -p $GITHUB_WORKSPACE/build
export PKG_CONFIG_PATH=/opt/homebrew/Cellar/libxml2/lib/pkgconfig
# FIXME: remove -DBUILD_PYTHON_BINDINGS=OFF. Python tests fail with "ModuleNotFoundError: No module named '_gdal'" with macos-12
cmake -Werror=dev -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal ${CMAKE_OPTIONS} -DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror -DUSE_CCACHE=ON -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DWERROR_DEV_FLAG="-Werror=dev" -DBUILD_PYTHON_BINDINGS=OFF
#$(cat $GITHUB_WORKSPACE/build/python_executable.txt) -m pip install --user -r $GITHUB_WORKSPACE/autotest/requirements.txt
- name: Build
run: |
cmake --build $GITHUB_WORKSPACE/build -j 3
otool -L $GITHUB_WORKSPACE/build/libgdal.dylib
- name: test
run: cmake --build $GITHUB_WORKSPACE/build --target quicktest
- name: test (with ctest)
run: |
ctest --test-dir $GITHUB_WORKSPACE/build -V -j 3
env:
BUILD_NAME: "macos_build_brew"
- name: Install
run: cmake --build $GITHUB_WORKSPACE/build --target install -j 3
- name: Test post-install usage (with pkg-config)
run: |
./autotest/postinstall/test_pkg-config.sh $GITHUB_WORKSPACE/install-gdal
- name: Test post-install usage (with gdal-config)
run: |
./autotest/postinstall/test_gdal-config.sh $GITHUB_WORKSPACE/install-gdal
- name: Test post-install usage (with CMake)
run: |
./autotest/postinstall/test_cmake.sh $GITHUB_WORKSPACE/install-gdal --dynamic --disable-odbc
- name: ccache statistics
run: ccache -s
# Disabled for now since bindings in master use GDALDatasetGetFieldDomainNames
# but libgdal from Conda has not it yet.
build-windows-conda-csharp-only:
if: ${{ false }} # disable for now
runs-on: windows-2022
env:
architecture: x64
generator: Visual Studio 17 2022
steps:
# To avoid git clone to mess with the line endings of GDAL autotest data
# files that look like text, but should be handled as binary content
- name: Set git core.autocrlf to false
run: |
git config --global core.autocrlf false
- name: Checkout GDAL
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
with:
activate-environment: gdalenv
python-version: 3.9
channels: conda-forge
- name: Install dependency
shell: bash -l {0}
run: |
conda install --yes --quiet --name gdalenv curl libiconv icu python=3.9 swig numpy pytest pytest-env pytest-benchmark filelock zlib clcache lxml
conda install --yes --quiet --name gdalenv -c conda-forge libgdal
- name: Configure
shell: bash -l {0}
run: |
mkdir -p $GITHUB_WORKSPACE/build
cmake -A ${architecture} -G "${generator}" -DGDAL_CSHARP_ONLY=ON -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build"
- name: Build
shell: bash -l {0}
run: cmake --build $GITHUB_WORKSPACE/build --config Release --target csharp_samples -j 2
- name: test (with ctest)
shell: bash -l {0}
run: |
ctest --test-dir $GITHUB_WORKSPACE/build -C Release -V -j 3 -R "^csharp.*"
env:
SKIP_OGR_GMLAS_HUGE_PROCESSING_TIME: YES