Distro #2437
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Distro | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| DEBUG_ENABLED: | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| type: boolean | |
| required: false | |
| default: false | |
| DEBUG_OS: | |
| description: 'which runner os to run the tmate action in (if the tmate action is run)' | |
| type: string | |
| default: 'windows-2019' | |
| required: false | |
| DEBUG_ARCH: | |
| description: 'which runner arch to run the tmate action in (if the tmate action is run)' | |
| type: string | |
| default: 'x86_64' | |
| required: false | |
| DEBUG_DETACHED: | |
| description: 'whether to launch tmate in detached mode (if the tmate action is run)' | |
| type: boolean | |
| required: false | |
| default: true | |
| LLVM_COMMIT: | |
| description: 'llvm commit to build' | |
| type: string | |
| required: false | |
| default: '' | |
| CMAKE_ARGS: | |
| description: 'extra cmake args' | |
| type: string | |
| required: false | |
| default: '' | |
| # why are these strings and not type boolean? | |
| # https://stackoverflow.com/a/76294014/9045206 | |
| APPLY_PATCHES: | |
| description: 'whether to apply patches to source' | |
| required: false | |
| default: true | |
| type: boolean | |
| USE_CMAKE_NAMESPACES: | |
| description: 'whether to build with cmake namespaced targets' | |
| required: false | |
| default: false | |
| type: boolean | |
| BUILD_NATIVE_TOOLS: | |
| description: 'whether to build native tools' | |
| required: false | |
| default: true | |
| type: boolean | |
| RUN_TESTS: | |
| description: 'whether to run llvm tests after build' | |
| required: false | |
| default: false | |
| type: boolean | |
| # DEBUG_CI_FAST_BUILD: | |
| # description: 'whether to build only one single llvm target (llvm-tblgen)' | |
| # required: false | |
| # default: false | |
| # type: boolean | |
| pull_request: | |
| release: | |
| types: | |
| - published | |
| schedule: | |
| # At minute 0 past hour 1. (see https://crontab.guru) | |
| - cron: '00 01 * * *' | |
| # At minute 0 past every 4th hour. (see https://crontab.guru) | |
| - cron: '0 */4 * * *' | |
| jobs: | |
| get_llvm_project_commit: | |
| name: Get latest LLVM commit | |
| runs-on: ubuntu-latest | |
| outputs: | |
| LLVM_PROJECT_COMMIT: ${{ steps.get_llvm_project_commit.outputs.LLVM_PROJECT_COMMIT }} | |
| DATETIME: ${{ steps.get_llvm_project_commit.outputs.DATETIME }} | |
| steps: | |
| - name: Get llvm-project commit | |
| id: get_llvm_project_commit | |
| run: | | |
| if [ x"${{ inputs.LLVM_COMMIT }}" == x"" ]; then | |
| sudo apt install jq | |
| LLVM_PROJECT_COMMIT=$(curl -s https://api.github.com/repos/llvm/llvm-project/commits/main | jq -r '.sha[:8]') | |
| else | |
| LLVM_PROJECT_COMMIT="${{ inputs.LLVM_COMMIT }}" | |
| fi | |
| echo "LLVM_PROJECT_COMMIT=${LLVM_PROJECT_COMMIT}" | tee -a $GITHUB_OUTPUT | |
| DATETIME=$(date +"%Y%m%d%H") | |
| echo "DATETIME=${DATETIME}" | tee -a $GITHUB_OUTPUT | |
| settings: | |
| runs-on: ubuntu-latest | |
| needs: get_llvm_project_commit | |
| steps: | |
| - name: Set commons as env vars | |
| id: settings | |
| run: | | |
| UPLOAD_ARTIFACTS="true" | |
| if [ x"${{ github.event.schedule }}" == x"0 */4 * * *" ] || [ x"$DEBUG_CI_FAST_BUILD" == x"true" ]; then | |
| UPLOAD_ARTIFACTS="false" | |
| fi | |
| echo "UPLOAD_ARTIFACTS=$UPLOAD_ARTIFACTS" | tee -a $GITHUB_OUTPUT | |
| outputs: | |
| APPLY_PATCHES: ${{ inputs.APPLY_PATCHES || true }} | |
| BUILD_NATIVE_TOOLS: ${{ inputs.BUILD_NATIVE_TOOLS || true }} | |
| CMAKE_ARGS: ${{ inputs.CMAKE_ARGS }} | |
| DATETIME: ${{ needs.get_llvm_project_commit.outputs.DATETIME }} | |
| DEBUG_ARCH: ${{ inputs.DEBUG_ARCH }} | |
| DEBUG_DETACHED: ${{ inputs.DEBUG_DETACHED }} | |
| DEBUG_ENABLED: ${{ inputs.DEBUG_ENABLED }} | |
| DEBUG_OS: ${{ inputs.DEBUG_OS }} | |
| LLVM_PROJECT_COMMIT: ${{ needs.get_llvm_project_commit.outputs.LLVM_PROJECT_COMMIT }} | |
| RUN_TESTS: ${{ inputs.RUN_TESTS }} | |
| UPLOAD_ARTIFACTS: ${{ steps.settings.outputs.UPLOAD_ARTIFACTS }} | |
| USE_CMAKE_NAMESPACES: ${{ inputs.USE_CMAKE_NAMESPACES }} | |
| build: | |
| needs: settings | |
| name: ${{ matrix.OS }} ${{ matrix.ARCH }} ${{ matrix.BUILD_CUDA && 'cuda' || '' }} ${{ matrix.BUILD_AMDGPU && 'amdgpu' || '' }} ${{ matrix.BUILD_VULKAN && 'vulkan' || '' }} ${{ matrix.BUILD_OPENMP && 'openmp' || '' }} | |
| continue-on-error: true | |
| runs-on: ${{ matrix.OS }} | |
| outputs: | |
| MLIR_WHEEL_VERSION: ${{ steps.get_wheel_version.outputs.MLIR_WHEEL_VERSION }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - OS: ubuntu-20.04 | |
| ARCH: x86_64 | |
| - OS: ubuntu-22.04 | |
| ARCH: wasm32 | |
| - OS: ubuntu-20.04 | |
| ARCH: aarch64 | |
| - OS: ubuntu-20.04 | |
| ARCH: x86_64 | |
| BUILD_CUDA: true | |
| - OS: ubuntu-20.04 | |
| ARCH: x86_64 | |
| BUILD_AMDGPU: true | |
| # - OS: ubuntu-20.04 | |
| # ARCH: x86_64 | |
| # BUILD_VULKAN: true | |
| # - OS: ubuntu-20.04 | |
| # ARCH: x86_64 | |
| # BUILD_OPENMP: true | |
| - OS: windows-2019 | |
| ARCH: AMD64 | |
| - OS: macos-12 | |
| ARCH: x86_64 | |
| - OS: macos-12 | |
| ARCH: arm64 | |
| # - OS: macos-12 | |
| # ARCH: arm64 | |
| # BUILD_VULKAN: true | |
| # | |
| # - OS: macos-12 | |
| # ARCH: arm64 | |
| # BUILD_OPENMP: true | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - name: Checkout actions | |
| uses: actions/checkout@v3 | |
| - uses: ./actions/setup_base | |
| id: setup_base | |
| with: | |
| # optional | |
| DEBUG_ENABLED: ${{ needs.settings.outputs.DEBUG_ENABLED }} | |
| DEBUG_OS: ${{ needs.settings.outputs.DEBUG_OS }} | |
| DEBUG_ARCH: ${{ needs.settings.outputs.DEBUG_ARCH }} | |
| DEBUG_DETACHED: ${{ needs.settings.outputs.DEBUG_DETACHED }} | |
| # required | |
| MATRIX_OS: ${{ matrix.OS }} | |
| MATRIX_ARCH: ${{ matrix.ARCH }} | |
| - uses: ./actions/setup_ccache | |
| id: setup_ccache | |
| with: | |
| MATRIX_OS: ${{ matrix.OS }} | |
| MATRIX_ARCH: ${{ matrix.ARCH }} | |
| EXTRA_KEY: ${{ matrix.BUILD_AMDGPU && 'amdgpu-' || '' }}${{ matrix.BUILD_CUDA && 'cuda-' || '' }}${{ matrix.BUILD_VULKAN && 'vulkan-' || '' }}${{ matrix.BUILD_OPENMP && 'openmp' || '' }} | |
| WORKSPACE_ROOT: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| # This step is needed because action/checkout does not support paths. | |
| # don't move because you can't parameterize -uses: ./actions/setup_ccache with workspace_root | |
| - name: Copy stuff to workspace_root | |
| run: | | |
| WORKSPACE_ROOT="${{ steps.setup_base.outputs.WORKSPACE_ROOT }}" | |
| if [[ "${{ matrix.OS }}" == *"windows"* ]]; then | |
| WORKSPACE_ROOT=$(echo "/$WORKSPACE_ROOT" | sed -e 's/\\/\//g' -e 's/://') | |
| fi | |
| ls -lah | |
| if [[ x"$PWD" != x"$WORKSPACE_ROOT" ]]; then | |
| shopt -s dotglob | |
| cp -r ./* "$WORKSPACE_ROOT" | |
| fi | |
| ls -lah "$WORKSPACE_ROOT" | |
| # setup llvm | |
| - name: Get LLVM | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| curl -s https://codeload.github.com/llvm/llvm-project/zip/${{ needs.settings.outputs.LLVM_PROJECT_COMMIT }} -o llvm.zip | |
| unzip -o -q llvm.zip | |
| rm -rf llvm.zip | |
| LLVM_PROJECT_COMMIT=${{ needs.settings.outputs.LLVM_PROJECT_COMMIT }} | |
| mv llvm-project-$LLVM_PROJECT_COMMIT llvm-project | |
| - name: Set env variables | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| echo "APPLY_PATCHES=${{ needs.settings.outputs.APPLY_PATCHES }}" >> $GITHUB_ENV | |
| echo "CMAKE_ARGS=${{ needs.settings.outputs.CMAKE_ARGS }}" >> $GITHUB_ENV | |
| echo "RUN_TESTS=${{ needs.settings.outputs.RUN_TESTS }}" >> $GITHUB_ENV | |
| echo "DATETIME=${{ needs.settings.outputs.DATETIME }}" >> $GITHUB_ENV | |
| echo "LLVM_PROJECT_COMMIT=${{ needs.settings.outputs.LLVM_PROJECT_COMMIT }}" >> $GITHUB_ENV | |
| echo "USE_CMAKE_NAMESPACES=${{ needs.settings.outputs.USE_CMAKE_NAMESPACES }}" >> $GITHUB_ENV | |
| echo "BUILD_CUDA=${{ matrix.BUILD_CUDA }}" >> $GITHUB_ENV | |
| echo "BUILD_AMDGPU=${{ matrix.BUILD_AMDGPU }}" >> $GITHUB_ENV | |
| echo "BUILD_OPENMP=${{ matrix.BUILD_OPENMP }}" >> $GITHUB_ENV | |
| echo "BUILD_VULKAN=${{ matrix.BUILD_VULKAN }}" >> $GITHUB_ENV | |
| echo "CIBW_ARCHS=${{ matrix.ARCH }}" >> $GITHUB_ENV | |
| echo "MATRIX_OS=${{ matrix.OS }}" >> $GITHUB_ENV | |
| echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV | |
| echo "HOST_CCACHE_DIR="$(ccache --get-config cache_dir)"" >> $GITHUB_ENV | |
| echo "PARALLEL_LEVEL=2" >> $GITHUB_ENV | |
| echo "PIP_FIND_LINKS=https://github.com/makslevental/mlir-wheels/releases/expanded_assets/latest" >> $GITHUB_ENV | |
| - name: set ENV macos | |
| if: contains(matrix.OS, 'macos') | |
| shell: bash | |
| run: | | |
| # echo "MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion)" | tee -a $GITHUB_ENV | |
| echo "MACOSX_DEPLOYMENT_TARGET=11.0" | tee -a $GITHUB_ENV | |
| # build | |
| - name: cibuildwheel | |
| if: ${{ matrix.ARCH != 'aarch64' && matrix.ARCH != 'wasm32' }} | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| cibuildwheel --output-dir wheelhouse | |
| - name: build aarch ubuntu wheel | |
| if: ${{ matrix.OS == 'ubuntu-20.04' && matrix.ARCH == 'aarch64' }} | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| ./scripts/apply_patches.sh | |
| export RUN_TESTS=0 | |
| pip wheel . -v -w wheelhouse | |
| - name: Setup Emscripten | |
| if: contains(matrix.ARCH, 'wasm') | |
| uses: mymindstorm/setup-emsdk@v14 | |
| with: | |
| version: 3.1.60 | |
| - name: build wasm ubuntu wheel | |
| if: contains(matrix.ARCH, 'wasm') | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| ./scripts/apply_patches.sh | |
| $EMSDK/emsdk activate 3.1.60 | |
| source $EMSDK/emsdk_env.sh | |
| - name: build wasm ubuntu wheel | |
| if: contains(matrix.ARCH, 'wasm') | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| # [scalar content] can be written with the folded style (denoted by “>”) | |
| # where each line break is folded to a space unless it ends an empty or a more-indented line. | |
| run: >- | |
| cmake -G Ninja | |
| -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake | |
| -DLLVM_BUILD_TOOLS="OFF" | |
| -DLLVM_INCLUDE_UTILS="OFF" | |
| -DLLVM_INCLUDE_EXAMPLES="OFF" | |
| -DLLVM_INCLUDE_TESTS="OFF" | |
| -DLLVM_INCLUDE_BENCHMARKS="OFF" | |
| -DLLVM_ENABLE_BINDINGS="OFF" | |
| -DLLVM_BUILD_DOCS="OFF" | |
| -DLLVM_ENABLE_OCAMLDOC="OFF" | |
| -DLLVM_ENABLE_THREADS="OFF" | |
| -DLLVM_ENABLE_BACKTRACES="OFF" | |
| -DLLVM_ENABLE_PIC="OFF" | |
| -DLLVM_ENABLE_UNWIND_TABLES="OFF" | |
| -DLLVM_ENABLE_CRASH_OVERRIDES="OFF" | |
| -DLLVM_ENABLE_TERMINFO="OFF" | |
| -DLLVM_ENABLE_LIBXML2="OFF" | |
| -DLLVM_ENABLE_LIBEDIT="OFF" | |
| -DLLVM_ENABLE_LIBPFM="OFF" | |
| -DLLVM_ENABLE_ZLIB="OFF" | |
| -DLLVM_ENABLE_ZSTD="OFF" | |
| -DLLVM_ENABLE_PROJECTS="mlir;llvm" | |
| -DCMAKE_BUILD_TYPE="Release" | |
| -DLLVM_TARGETS_TO_BUILD="WebAssembly" | |
| -DLLVM_TARGET_ARCH="wasm32-wasi" | |
| -DLLVM_DEFAULT_TARGET_TRIPLE="wasm32-wasi" | |
| -DLLVM_HAVE_LIBXAR="OFF" | |
| -DCMAKE_SYSTEM_NAME=Emscripten | |
| -DLLVM_HOST_TRIPLE=x86_64-unknown-linux-gnu | |
| -DCROSS_TOOLCHAIN_FLAGS_NATIVE="-DCMAKE_C_COMPILER=gcc;-DCMAKE_CXX_COMPILER=g++" | |
| -DCMAKE_EXE_LINKER_FLAGS="-sSTANDALONE_WASM=1 -sWASM=1 -sWASM_BIGINT=1 -sEXPORT_ALL=1 -sMODULARIZE=1 -sEXPORT_NAME=MLIRModule -sEXPORTED_RUNTIME_METHODS=cwrap,ccall,getValue,setValue,writeAsciiToMemory,wasmTable --minify 0 -lembind" | |
| -B$PWD/build_mlir_wasm | |
| -S$PWD/llvm-project/llvm | |
| - name: build wasm ubuntu wheel | |
| if: contains(matrix.ARCH, 'wasm') | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| cmake --build build_mlir_wasm --target mlir-opt | |
| mkdir wheelhouse | |
| zip wheelhouse/mlir_wasm-$(python setup.py --mlir-version).zip build_mlir_wasm/bin/*.js build_mlir_wasm/bin/*.wasm -v | |
| rm -rf build_mlir_wasm | |
| - name: Clean llvm-project | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| rm -rf llvm-project | |
| rm -rf build | |
| - name: Get wheel version | |
| if: ${{ !contains(matrix.ARCH, 'wasm') }} | |
| id: get_wheel_version | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| pip install pkginfo | |
| WHL=$(ls wheelhouse/mlir-*whl) | |
| echo "MLIR_WHEEL_VERSION=$(python -c "import pkginfo; w = pkginfo.Wheel('$WHL'); print(w.version.split('+')[0] + '+' + w.version.split('+')[1].rsplit('.', 1)[-1])")" | tee -a $GITHUB_OUTPUT | |
| - name: Download cache from container ubuntu | |
| if: (matrix.OS == 'ubuntu-20.04' && matrix.ARCH == 'x86_64') && (success() || failure()) | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| ccache -s | |
| rm -rf $HOST_CCACHE_DIR | |
| mv ./wheelhouse/.ccache $HOST_CCACHE_DIR | |
| ls -la $HOST_CCACHE_DIR | |
| ccache -s | |
| - name: Reset datetime ccache | |
| # find: The environment is too large for exec(). | |
| if: ${{ !contains(matrix.OS, 'windows') }} | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| ccache --print-stats | |
| find $HOST_CCACHE_DIR -exec touch -a -m -t 197001010000 {} \; | |
| - name: rename non-windows | |
| if: ${{ matrix.OS == 'ubuntu-20.04' || matrix.OS == 'macos-12' }} | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| rename 's/cp310-cp310/py3-none/' wheelhouse/mlir-*whl | |
| rename 's/cp311-cp311/py3-none/' wheelhouse/mlir-*whl | |
| if [ x"${{ matrix.OS }}" == x"ubuntu-20.04" ] && [ x"${{ matrix.ARCH }}" == x"aarch64" ]; then | |
| rename 's/x86_64/aarch64/' wheelhouse/mlir-*whl | |
| fi | |
| # git-bash doesn't have rename | |
| - name: rename windows | |
| shell: pwsh | |
| if: ${{ matrix.OS == 'windows-2019' }} | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| ls wheelhouse/mlir-*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' } | |
| ls wheelhouse/mlir-*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' } | |
| - name: Build native_tools wheel | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| id: build_native_tools_wheel | |
| if: ${{ !contains(matrix.ARCH, 'wasm') && needs.settings.outputs.BUILD_NATIVE_TOOLS == 'true' }} | |
| run: | | |
| for TOOL in "llvm-tblgen" "mlir-tblgen" "mlir-linalg-ods-yaml-gen" "mlir-pdll" "llvm-config" "FileCheck"; do | |
| if [ x"${{ matrix.OS }}" == x"windows-2019" ]; then | |
| TOOL="$TOOL.exe" | |
| fi | |
| unzip -j wheelhouse/mlir-*whl "mlir/bin/$TOOL" -d native_tools/ | |
| done | |
| if [ x"${{ matrix.OS }}" == x"ubuntu-20.04" ]; then | |
| PLAT="linux" | |
| elif [ x"${{ matrix.OS }}" == x"macos-12" ]; then | |
| PLAT="macosx_12_0" | |
| elif [ x"${{ matrix.OS }}" == x"windows-2019" ]; then | |
| PLAT="win" | |
| fi | |
| PLAT=${PLAT}_$(echo ${{ matrix.ARCH }} | tr '[:upper:]' '[:lower:]') | |
| pushd native_tools | |
| MLIR_WHEEL_VERSION=${{ steps.get_wheel_version.outputs.MLIR_WHEEL_VERSION }} \ | |
| python setup.py bdist_wheel --dist-dir ../wheelhouse --plat $PLAT | |
| popd | |
| - name: Release current commit | |
| if: ${{ needs.settings.outputs.UPLOAD_ARTIFACTS == 'true' }} | |
| uses: ncipollo/[email protected] | |
| with: | |
| artifacts: | | |
| ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}/wheelhouse/*.whl | |
| ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}/wheelhouse/*.zip | |
| token: "${{ secrets.GITHUB_TOKEN }}" | |
| tag: "latest" | |
| name: "latest" | |
| removeArtifacts: false | |
| allowUpdates: true | |
| replacesArtifacts: true | |
| makeLatest: true | |
| - name: Release current commit | |
| if: ${{ needs.settings.outputs.UPLOAD_ARTIFACTS == 'true' }} | |
| uses: ncipollo/[email protected] | |
| with: | |
| owner: makslevental | |
| repo: wheels | |
| artifacts: | | |
| ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}/wheelhouse/*.whl | |
| ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}/wheelhouse/*.zip | |
| token: "${{ secrets.WHEELS_REPO }}" | |
| tag: "i" | |
| name: "i" | |
| removeArtifacts: false | |
| allowUpdates: true | |
| replacesArtifacts: true | |
| makeLatest: true | |
| call-build-python-bindings: | |
| needs: [build, settings] | |
| if: ${{ needs.settings.outputs.UPLOAD_ARTIFACTS == 'true' }} | |
| permissions: | |
| contents: write | |
| id-token: write | |
| # DON'T FORGET TO CHANGE THIS IF YOU UPDATE WHEELS.YML | |
| uses: makslevental/mlir-wheels/.github/workflows/wheels.yml@main | |
| secrets: inherit # pass all secrets | |
| with: | |
| MLIR_WHEEL_VERSION: ${{ needs.build.outputs.MLIR_WHEEL_VERSION }} |