Merge pull request #458 from GATEOverflow/mlperf-inference

Support test_input_index and test_input_id to customise CM test script
mlcommons · Oct 31, 2024 · 1f9df67 · 1f9df67
2 parents f34ef4b + f467f5c
commit 1f9df67
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 2 deletions.
diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml
@@ -11,6 +11,10 @@ on:
 jobs:
   run-script-tests:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test-input-index: [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11+" ]
     steps:
     - name: 'Checkout'
       uses: actions/checkout@v4
@@ -30,4 +34,4 @@ jobs:
         done
         python3 -m pip install cmind
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
-        DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }}
+        DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }}
diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml
@@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations
 
 on:
   schedule:
-    - cron: "59 16 * * *" #to be adjusted
+    - cron: "31 2 * * *" #to be adjusted
 
 jobs:
   build_nvidia:

diff --git a/automation/script/module.py b/automation/script/module.py
@@ -2432,8 +2432,34 @@ def test(self, i):
                     logging.info(test_config)
                     variations = meta.get("variations")
                     tags_string = ",".join(meta.get("tags"))
+                    test_input_index = i.get('test_input_index')
+                    test_input_id = i.get('test_input_id')
                     run_inputs = i.get("run_inputs", test_config.get('run_inputs', [ {"docker_os": "ubuntu", "docker_os_version":"22.04"} ]))
+                    if test_input_index:
+                        index_plus = False
+                        try:
+                            if test_input_index.endswith("+"):
+                                input_index = int(test_input_index[:-1])
+                                index_plus = True
+                            else:
+                                input_index = int(test_input_index)
+                        except ValueError as e:
+                            print(e)
+                            return {'return': 1, 'error': f'Invalid test_input_index: {test_input_index}. Must be an integer or an integer followed by a +'}
+                        if input_index > len(run_inputs):
+                            run_inputs = []
+                        else:
+                            if index_plus:
+                                run_inputs = run_inputs[index_index-1:]
+                            else:
+                                run_inputs = [ run_inputs[input_index - 1] ]
+
                     for run_input in run_inputs:
+                        if test_input_id:
+                            if run_input.get('id', '') != test_input_id:
+                                continue
+
+
                         ii = {'action': 'run',
                               'automation':'script',
                               'quiet': i.get('quiet'),

diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py
@@ -30,6 +30,8 @@
         ii['docker_cm_repo'] = os.environ['DOCKER_CM_REPO']
     if os.environ.get('DOCKER_CM_REPO_BRANCH', '') != '':
         ii['docker_cm_repo_branch'] = os.environ['DOCKER_CM_REPO_BRANCH']
+    if os.environ.get('TEST_INPUT_INDEX', '') != '':
+        ii['test_input_index'] = os.environ['TEST_INPUT_INDEX']
     print(ii)
     r = cm.access(ii)