Version v0.2.4 #3190
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench | |
| on: | |
| pull_request: | |
| branches: [main] | |
| merge_group: | |
| types: [checks_requested] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| bench-hyperfine: | |
| name: Hyperfine | |
| runs-on: ubuntu-24.04 | |
| env: | |
| CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse | |
| MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/ | |
| TABLEGEN_190_PREFIX: /usr/lib/llvm-19/ | |
| LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/ | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: check and free hdd space left | |
| run: | | |
| echo "Listing 20 largest packages" | |
| dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20 | |
| df -h | |
| sudo apt-get update | |
| sudo apt-get remove -y '^llvm-.*' | |
| sudo apt-get remove -y 'php.*' | |
| sudo apt-get remove -y '^dotnet-.*' | |
| sudo apt-get remove -y '^temurin-.*' | |
| sudo apt-get remove -y azure-cli microsoft-edge-stable google-chrome-stable firefox mono-devel | |
| sudo apt-get autoremove -y | |
| sudo apt-get clean | |
| df -h | |
| echo "Removing large directories" | |
| # deleting 15GB | |
| sudo rm -rf /usr/share/dotnet/ | |
| sudo rm -rf /usr/local/lib/android | |
| df -h | |
| - uses: dtolnay/[email protected] | |
| with: | |
| components: clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: add llvm deb repository | |
| uses: myci-actions/add-deb-repo@11 | |
| with: | |
| repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main | |
| repo-name: llvm-repo | |
| keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key | |
| - name: Install LLVM | |
| run: sudo apt-get install llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools | |
| - name: Install hyperfine | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: [email protected] | |
| - name: Install deps | |
| run: make deps | |
| - name: Run benchmarks | |
| run: make bench | |
| - name: Create markdown file | |
| run: bash .github/scripts/merge-benches.sh | |
| - name: Find Bench Comment | |
| continue-on-error: true | |
| uses: peter-evans/find-comment@v3 | |
| id: fc | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: "github-actions[bot]" | |
| body-includes: Benchmarking | |
| - name: Create or update bench comment | |
| continue-on-error: true | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-path: bench-hyperfine.md | |
| edit-mode: replace | |
| build-binaries: | |
| env: | |
| MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/ | |
| LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/ | |
| TABLEGEN_190_PREFIX: /usr/lib/llvm-19/ | |
| RUST_LOG: cairo_native=debug,cairo_native_test=debug | |
| strategy: | |
| matrix: | |
| branch: [ base, head ] | |
| name: Build Binary for ${{ matrix.branch }} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Cache binary | |
| uses: actions/cache@v3 | |
| id: cache-binary | |
| with: | |
| path: bin/cairo-native-run-${{ matrix.branch }} | |
| key: binary-${{ github.event.pull_request[matrix.branch].sha }} | |
| - name: Cache runtime-library | |
| uses: actions/cache@v3 | |
| id: cache-library | |
| with: | |
| path: lib/libcairo-native-runtime-${{ matrix.branch }}.a | |
| key: library-${{ github.event.pull_request[matrix.branch].sha }} | |
| - name: check and free hdd space left | |
| if: ${{ steps.cache-binary.outputs.cache-hit != 'true' }} | |
| run: | | |
| echo "Listing 20 largest packages" | |
| dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20 | |
| df -h | |
| sudo apt-get update | |
| sudo apt-get remove -y '^llvm-.*' | |
| sudo apt-get remove -y 'php.*' | |
| sudo apt-get remove -y '^dotnet-.*' | |
| sudo apt-get remove -y '^temurin-.*' | |
| sudo apt-get remove -y azure-cli microsoft-edge-stable google-chrome-stable firefox mono-devel | |
| sudo apt-get autoremove -y | |
| sudo apt-get clean | |
| df -h | |
| echo "Removing large directories" | |
| # deleting 15GB | |
| sudo rm -rf /usr/share/dotnet/ | |
| sudo rm -rf /usr/local/lib/android | |
| df -h | |
| - name: Install Rust | |
| if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} | |
| uses: dtolnay/[email protected] | |
| - name: add llvm deb repository | |
| if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} | |
| uses: myci-actions/add-deb-repo@11 | |
| with: | |
| repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main | |
| repo-name: llvm-repo | |
| keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key | |
| - name: Install LLVM | |
| if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} | |
| run: sudo apt-get update && sudo apt-get install llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools | |
| - name: Checkout base commit | |
| if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request[matrix.branch].sha }} | |
| - name: Fetch Rust cache | |
| if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Build binary | |
| if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} | |
| run: | | |
| make build | |
| mkdir lib bin | |
| cp target/release/cairo-native-run bin/cairo-native-run-${{ matrix.branch }} | |
| cp target/release/libcairo_native_runtime.a lib/libcairo-native-runtime-${{ matrix.branch }}.a | |
| run-bench: | |
| env: | |
| MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/ | |
| LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/ | |
| TABLEGEN_190_PREFIX: /usr/lib/llvm-19/ | |
| RUST_LOG: cairo_native=debug,cairo_native_test=debug | |
| strategy: | |
| matrix: | |
| branch: [ base, head ] | |
| name: Run benches for ${{ matrix.branch }} | |
| runs-on: ubuntu-24.04 | |
| needs: [ build-binaries ] | |
| steps: | |
| - name: Checkout base commit | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request[matrix.branch].sha }} | |
| - name: Install Hyperfine | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: [email protected] | |
| - name: Fetch ${{ matrix.branch }} binary | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: bin/cairo-native-run-${{ matrix.branch }} | |
| key: binary-${{ github.event.pull_request[matrix.branch].sha }} | |
| - name: Fetch ${{ matrix.branch }} runtime-library | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: lib/libcairo-native-runtime-${{ matrix.branch }}.a | |
| key: library-${{ github.event.pull_request[matrix.branch].sha }} | |
| - name: Run Program | |
| run: | | |
| make deps | |
| mkdir bench-results-${{ matrix.branch }} | |
| cp programs/benches/*.cairo bench-results-${{ matrix.branch }} | |
| for f in bench-results-${{ matrix.branch }}/*.cairo; do | |
| hyperfine \ | |
| --show-output \ | |
| --export-markdown "${f%.*}.md" \ | |
| --export-json "${f%.*}.json" \ | |
| --warmup 3 \ | |
| -n "${{matrix.branch}} $(basename $f) (JIT)" "./bin/cairo-native-run-${{ matrix.branch }} --run-mode=jit -s $f --opt-level 3 --available-gas 18446744073709551615" \ | |
| -n "${{matrix.branch}} $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-${{ matrix.branch }}.a ./bin/cairo-native-run-${{ matrix.branch }} --run-mode=aot -s $f --opt-level 3 --available-gas 18446744073709551615" \ | |
| >> /dev/stderr | |
| done | |
| - name: Upload benches ${{matrix.branch}} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{matrix.branch}} | |
| path: bench-results-${{matrix.branch}} | |
| show-outputs: | |
| name: Print Bench Results | |
| runs-on: ubuntu-24.04 | |
| needs: [ run-bench ] | |
| steps: | |
| - name: Fetch Base Result | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: results-base | |
| path: bench-results-base | |
| - name: Fetch Head Result | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: results-head | |
| path: bench-results-head | |
| - name: Print tables | |
| run: | | |
| { | |
| echo "${bold}Benchmark results Main vs HEAD.${normal}" | |
| for f in bench-results-base/*.md; do | |
| echo | |
| cat $f | |
| cat bench-results-head/$(basename $f) | |
| done | |
| } | tee -a comment_body.md | |
| - name: Find Bench Comment | |
| continue-on-error: true | |
| uses: peter-evans/find-comment@v3 | |
| id: fc | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: "github-actions[bot]" | |
| body-includes: Benchmark results Main vs HEAD | |
| - name: Create or update bench comment | |
| continue-on-error: true | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-path: comment_body.md | |
| edit-mode: replace |