Fix daily block run (#891)

* Run on pull_request (tmp) * Use paralelism * Fix paths * Remove old compare step * Fix paths * Save output to env var * Add comments * Create issue * Add workflow url * Mock for testing * Print env vars * Save output * Mock issue * Update template * Tee compare output * Remove mock * Make run for testing * Improve template * Increase range * Update script message * Update issue message * Add always * Change message * Prevent upload error * Add comment * Add comment * Add comments and continue-on-error * Rename dump * Comment blocks * Increase size * Don't run on PR * Improve comme * Remove todo! --------- Co-authored-by: Pedro Fontana <[email protected]>
lambdaclass · Nov 5, 2024 · 9791075 · 9791075
1 parent 742a6b1
commit 9791075
Show file tree

Hide file tree

Showing 3 changed files with 109 additions and 7 deletions.
diff --git a/.github/ISSUE_TEMPLATE/daily_failure.md b/.github/ISSUE_TEMPLATE/daily_failure.md
@@ -0,0 +1,19 @@
+---
+title: "Daily Workflow Failure: Block Range {{ env.BLOCK_START }}-{{ env.BLOCK_END }}"
+labels: bug
+---
+
+Comparing VM execution against Native in the given block range produced diffs:
+
+- Commit: {{ env.COMMIT_SHA }}
+- Block Start: {{ env.BLOCK_START }}
+- Block End: {{ env.BLOCK_END }}
+- Workflow URL: {{ env.WORKFLOW_URL }}
+
+## Compare Output
+
+The transaction were not compared in order. You should rerun the whole block to find the error root
+
+```
+{{ env.OUTPUT }}
+```
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
@@ -6,13 +6,12 @@ on:
   - cron: "0 0 * * *"
 
 env:
-  RANGE_SIZE: 50
+  RANGE_SIZE: 25
   SEQUENCER_REV: 1b1b95cae7ae07b9bc778443ca75ee18008a6bc8
 
 jobs:
-  run-and-compare:
+  run:
     runs-on: ubuntu-latest
-    timeout-minutes: 1440
     env:
       LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
       MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
@@ -32,6 +31,9 @@ jobs:
           - 747000
           - 748000
           - 749000
+        runner:
+          - native
+          - vm
       fail-fast: false
     defaults:
       run:
@@ -79,17 +81,95 @@ jobs:
           grep $GIT Cargo.toml
           
       - name: Run with Native
+        if: ${{ matrix.runner == 'native' }}
         run: |
           BLOCK_START=${{ matrix.block }}
           BLOCK_END=$(($BLOCK_START + $RANGE_SIZE - 1))
           cargo run --release --features state_dump block-range $BLOCK_START $BLOCK_END mainnet
-        continue-on-error: true
       - name: Run with VM
+        if: ${{ matrix.runner == 'vm' }}
         run: |
           BLOCK_START=${{ matrix.block }}
           BLOCK_END=$(($BLOCK_START + $RANGE_SIZE - 1))
           cargo run --release --features state_dump,only_cairo_vm block-range $BLOCK_START $BLOCK_END mainnet
+
+      # We always upload the dump, even if the job fails
+      - name: Upload dumps
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: dump-${{matrix.block}}-${{matrix.runner}}
+          path: starknet-replay/state_dumps/${{matrix.runner}}
+
+
+  compare:
+    needs: [run]
+    runs-on: ubuntu-latest
+    # We always run the compare job, to ensure that a single run job failing
+    # would not cancel the whole comparison.
+    if: ${{ always() }}
+    strategy:
+      matrix:
+        block:
+          - 740000
+          - 741000
+          - 742000
+          - 743000
+          - 744000
+          - 745000
+          - 746000
+          - 747000
+          - 748000
+          - 749000
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fetch Native dumps
+        uses: actions/download-artifact@v4
+        with:
+          name: dump-${{matrix.block}}-native
+          path: state_dumps/native
+        continue-on-error: true
+      - name: Fetch VM dumps
+        uses: actions/download-artifact@v4
+        with:
+          name: dump-${{matrix.block}}-vm
+          path: state_dumps/vm
         continue-on-error: true
 
       - name: Compare states
-        run: ../cairo_native/scripts/cmp_state_dumps.sh
+        run: |
+          ./scripts/cmp_state_dumps.sh | tee output
+
+      - name: Prepare env vars
+        if: ${{ always() }}
+        run: |
+          # Save blocks to env var
+          BLOCK_START=${{ matrix.block }}
+          BLOCK_END=$(($BLOCK_START + $RANGE_SIZE - 1))
+          echo "BLOCK_END=$BLOCK_END" | tee -a $GITHUB_ENV
+          echo "BLOCK_START=$BLOCK_START" | tee -a $GITHUB_ENV
+
+          # Save workflow url
+          REPO_URL="${{ github.server_url }}/${{ github.repository }}"
+          echo "WORKFLOW_URL=$REPO_URL/actions/runs/${{ github.run_id }}" | tee -a $GITHUB_ENV
+
+          # Save output
+          {
+            echo 'OUTPUT<<EOF'
+            cat output
+            echo EOF
+          } >> "$GITHUB_ENV"
+
+      - name: Create Issue
+        if: ${{ failure() }}
+        uses: JasonEtco/create-an-issue@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMIT_SHA: ${{ github.sha }}
+        with:
+          filename: .github/ISSUE_TEMPLATE/daily_failure.md
diff --git a/scripts/cmp_state_dumps.sh b/scripts/cmp_state_dumps.sh
@@ -24,13 +24,16 @@ for vm_dump in state_dumps/vm/*/*.json; do
     continue
   fi
 
-  base=$(basename "$vm_dump")
+  tx_name=$(basename "$vm_dump")
+  tx=${tx_name//.*/}
+  block_name=$(basename "$(dirname "$vm_dump")")
+  block=${block_name//block/}
 
   if ! cmp -s \
       <(sed '/"reverted": /d' "$native_dump") \
       <(sed '/"reverted": /d' "$vm_dump")
   then
-    echo "diff:  $base"
+    echo "Diff at block $block, tx $tx"
     diffing=$((diffing+1))
   else
     matching=$((matching+1))