Steel #615
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Steel | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| branches: | |
| - main | |
| env: | |
| MAX_JOBS: 64 | |
| MIN_PROJECTS_PER_JOB: 4 | |
| MIN_PROJECTS_FOR_MATRIX: 4 | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| changed_projects: ${{ steps.analyze.outputs.changed_projects }} | |
| total_projects: ${{ steps.analyze.outputs.total_projects }} | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: changes | |
| if: github.event_name == 'pull_request' | |
| with: | |
| list-files: shell | |
| filters: | | |
| steel: | |
| - added|modified: '**/steel/**' | |
| workflow: | |
| - added|modified: '.github/workflows/steel.yml' | |
| - name: Analyze Changes | |
| id: analyze | |
| run: | | |
| # Generate ignore pattern, excluding comments | |
| ignore_pattern=$(grep -v '^#' .github/.ghaignore | grep -v '^$' | tr '\n' '|' | sed 's/|$//') | |
| echo "Ignore pattern: $ignore_pattern" | |
| function get_projects() { | |
| find . -type d -name "steel" | grep -vE "$ignore_pattern" | sort | |
| } | |
| # Determine which projects to build and test | |
| if [[ "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "schedule" || "${{ steps.changes.outputs.workflow }}" == "true" ]]; then | |
| projects=$(get_projects) | |
| elif [[ "${{ steps.changes.outputs.steel }}" == "true" ]]; then | |
| changed_files=(${{ steps.changes.outputs.steel_files }}) | |
| projects=$(for file in "${changed_files[@]}"; do dirname "${file}" | grep steel | sed 's#/steel/.*#/steel#g'; done | grep -vE "$ignore_pattern" | sort -u) | |
| else | |
| projects="" | |
| fi | |
| # Output project information | |
| if [[ -n "$projects" ]]; then | |
| echo "Projects to build and test" | |
| echo "$projects" | |
| total_projects=$(echo "$projects" | wc -l) | |
| echo "Total projects: $total_projects" | |
| echo "total_projects=$total_projects" >> $GITHUB_OUTPUT | |
| echo "changed_projects=$(echo "$projects" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT | |
| else | |
| echo "No projects to build and test." | |
| echo "total_projects=0" >> $GITHUB_OUTPUT | |
| echo "changed_projects=[]" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Generate matrix | |
| id: matrix | |
| run: | | |
| total_projects=${{ steps.analyze.outputs.total_projects }} | |
| max_jobs=${{ env.MAX_JOBS }} | |
| min_projects_per_job=${{ env.MIN_PROJECTS_PER_JOB }} | |
| min_projects_for_matrix=${{ env.MIN_PROJECTS_FOR_MATRIX }} | |
| if [ "$total_projects" -lt "$min_projects_for_matrix" ]; then | |
| echo "matrix=[0]" >> $GITHUB_OUTPUT | |
| else | |
| projects_per_job=$(( (total_projects + max_jobs - 1) / max_jobs )) | |
| projects_per_job=$(( projects_per_job > min_projects_per_job ? projects_per_job : min_projects_per_job )) | |
| num_jobs=$(( (total_projects + projects_per_job - 1) / projects_per_job )) | |
| indices=$(seq 0 $(( num_jobs - 1 ))) | |
| echo "matrix=[$(echo $indices | tr ' ' ',')]" >> $GITHUB_OUTPUT | |
| fi | |
| rust-checks: | |
| needs: changes | |
| if: ${{ github.event_name == 'pull_request' && needs.changes.outputs.total_projects != '0' }} | |
| name: Rust Checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: Run sccache-cache | |
| if: github.event_name != 'release' | |
| uses: mozilla-actions/[email protected] | |
| - name: Set Rust cache env vars | |
| if: github.event_name != 'release' | |
| run: | | |
| echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV | |
| echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV | |
| - name: Run fmt and clippy | |
| run: | | |
| readarray -t all_projects < <(echo '${{ needs.changes.outputs.changed_projects }}' | jq -r '.[]?') | |
| for project in "${all_projects[@]}"; do | |
| echo "::group::Checking ${project}" | |
| if [ ! -f "${project}/Cargo.toml" ]; then | |
| echo "::error::No Cargo.toml found in ${project}" | |
| exit 1 | |
| fi | |
| cd "${project}" | |
| cargo fmt --check | |
| cargo clippy --all-features -- -D warnings | |
| cd - > /dev/null | |
| echo "::endgroup::" | |
| done | |
| build-and-test: | |
| needs: changes | |
| if: needs.changes.outputs.total_projects != '0' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| index: ${{ fromJson(needs.changes.outputs.matrix) }} | |
| name: build-and-test-group-${{ matrix.index }} | |
| outputs: | |
| failed_projects: ${{ steps.set-failed.outputs.failed_projects }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - name: Run sccache-cache | |
| if: github.event_name != 'release' | |
| uses: mozilla-actions/[email protected] | |
| - name: Set Rust cache env vars | |
| if: github.event_name != 'release' | |
| run: | | |
| echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV | |
| echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cargo/bin/steel | |
| key: ${{ runner.os }}-steel-cli | |
| - name: Use Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20.x | |
| check-latest: true | |
| - name: Setup build environment | |
| id: setup | |
| run: | | |
| npm install --global pnpm | |
| # Create the build and test function | |
| cat << 'EOF' > build_and_test.sh | |
| function build_and_test() { | |
| local project=$1 | |
| local solana_version=$2 | |
| echo "Building and Testing $project with Solana $solana_version" | |
| cd "$project" || return 1 | |
| # Install dependencies | |
| if [ -f "package.json" ]; then | |
| if ! pnpm install --frozen-lockfile; then | |
| echo "::error::pnpm install failed for $project" | |
| echo "$project: pnpm install failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt | |
| cd - > /dev/null | |
| return 1 | |
| fi | |
| # Build | |
| if ! pnpm build; then | |
| echo "::error::build failed for $project" | |
| echo "$project: build failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt | |
| cd - > /dev/null | |
| return 1 | |
| fi | |
| # Test | |
| if ! pnpm build-and-test; then | |
| echo "::error::tests failed for $project" | |
| echo "$project: tests failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt | |
| cd - > /dev/null | |
| return 1 | |
| fi | |
| else | |
| # Use Steel CLI | |
| if ! cargo install --quiet steel-cli; then | |
| echo "::error::steel-cli installation failed for $project" | |
| echo "$project: steel-cli installation failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt | |
| cd - > /dev/null | |
| return 1 | |
| fi | |
| # Build | |
| if ! steel build; then | |
| echo "::error::steel build failed for $project" | |
| echo "$project: steel build failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt | |
| cd - > /dev/null | |
| return 1 | |
| fi | |
| # Test | |
| if ! steel test; then | |
| echo "::error::steel test failed for $project" | |
| echo "$project: steel test failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt | |
| cd - > /dev/null | |
| return 1 | |
| fi | |
| fi | |
| echo "Build and tests succeeded for $project with $solana_version version." | |
| cd - > /dev/null | |
| return 0 | |
| } | |
| function process_projects() { | |
| local solana_version=$1 | |
| readarray -t all_projects < <(echo '${{ needs.changes.outputs.changed_projects }}' | jq -r '.[]?') | |
| start_index=$(( ${{ matrix.index }} * ${{ env.MIN_PROJECTS_PER_JOB }} )) | |
| end_index=$(( start_index + ${{ env.MIN_PROJECTS_PER_JOB }} )) | |
| end_index=$(( end_index > ${{ needs.changes.outputs.total_projects }} ? ${{ needs.changes.outputs.total_projects }} : end_index )) | |
| echo "Projects to build and test in this job" | |
| for i in $(seq $start_index $(( end_index - 1 ))); do | |
| echo "${all_projects[$i]}" | |
| done | |
| failed=false | |
| for i in $(seq $start_index $(( end_index - 1 ))); do | |
| echo "::group::Building and testing ${all_projects[$i]}" | |
| if ! build_and_test "${all_projects[$i]}" "$solana_version"; then | |
| failed=true | |
| fi | |
| echo "::endgroup::" | |
| done | |
| return $([ "$failed" = true ] && echo 1 || echo 0) | |
| } | |
| EOF | |
| # Make the script executable | |
| chmod +x build_and_test.sh | |
| - name: Setup Solana stable | |
| uses: heyAyushh/[email protected] | |
| with: | |
| solana-cli-version: stable | |
| - name: Build and Test with Stable | |
| env: | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| run: | | |
| source build_and_test.sh | |
| solana -V | |
| rustc -V | |
| process_projects "stable" | |
| sccache --show-stats | |
| - name: Setup Solana 1.18.17 | |
| uses: heyAyushh/[email protected] | |
| with: | |
| solana-cli-version: 1.18.17 | |
| - name: Build and Test with 1.18.17 | |
| env: | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| run: | | |
| source build_and_test.sh | |
| solana -V | |
| rustc -V | |
| process_projects "1.18.17" | |
| sccache --show-stats | |
| - name: Set failed projects output | |
| id: set-failed | |
| if: failure() | |
| run: | | |
| if [ -f "$GITHUB_WORKSPACE/failed_projects.txt" ]; then | |
| failed_projects=$(cat $GITHUB_WORKSPACE/failed_projects.txt | jq -R -s -c 'split("\n")[:-1]') | |
| echo "failed_projects=$failed_projects" >> $GITHUB_OUTPUT | |
| else | |
| echo "failed_projects=[]" >> $GITHUB_OUTPUT | |
| fi | |
| summary: | |
| needs: [changes, build-and-test] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create job summary | |
| run: | | |
| echo "## Steel Workflow Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "- Total projects: ${{ needs.changes.outputs.total_projects }}" >> $GITHUB_STEP_SUMMARY | |
| # List all processed projects | |
| echo "<details>" >> $GITHUB_STEP_SUMMARY | |
| echo "<summary>Projects processed (click to expand)</summary>" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo '${{ needs.changes.outputs.changed_projects }}' | jq -r '.[]' | while read project; do | |
| echo "- $project" >> $GITHUB_STEP_SUMMARY | |
| done | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "</details>" >> $GITHUB_STEP_SUMMARY | |
| # Report build and test results | |
| if [[ "${{ needs.build-and-test.result }}" == "failure" ]]; then | |
| echo "## :x: Build or tests failed" >> $GITHUB_STEP_SUMMARY | |
| echo "<details>" >> $GITHUB_STEP_SUMMARY | |
| echo "<summary>Failed projects (click to expand)</summary>" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| failed_projects='${{ needs.build-and-test.outputs.failed_projects }}' | |
| if [[ -n "$failed_projects" ]]; then | |
| echo "$failed_projects" | jq -r '.[]' | while IFS=: read -r project failure_reason; do | |
| echo "- **$project**" >> $GITHUB_STEP_SUMMARY | |
| echo " - Failure reason: $failure_reason" >> $GITHUB_STEP_SUMMARY | |
| done | |
| else | |
| echo "No failed projects reported. This might indicate an unexpected error in the workflow." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "</details>" >> $GITHUB_STEP_SUMMARY | |
| elif [[ "${{ needs.build-and-test.result }}" == "success" ]]; then | |
| echo "## :white_check_mark: All builds and tests passed" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "## :warning: Build and test job was skipped or canceled" >> $GITHUB_STEP_SUMMARY | |
| fi |