diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml index ab2f5b3ab..e6c3db4b5 100644 --- a/.github/workflows/run-individual-script-tests.yml +++ b/.github/workflows/run-individual-script-tests.yml @@ -11,6 +11,10 @@ on: jobs: run-script-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-input-index: [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11+" ] steps: - name: 'Checkout' uses: actions/checkout@v4 @@ -30,4 +34,4 @@ jobs: done python3 -m pip install cmind cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} - DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} + DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 tests/script/process_tests.py ${{ steps.getfile.outputs.files }} diff --git a/.github/workflows/test-nvidia-mlperf-implementation.yml b/.github/workflows/test-nvidia-mlperf-implementation.yml index cb0b54978..032cb4624 100644 --- a/.github/workflows/test-nvidia-mlperf-implementation.yml +++ b/.github/workflows/test-nvidia-mlperf-implementation.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "59 16 * * *" #to be adjusted + - cron: "31 2 * * *" #to be adjusted jobs: build_nvidia: diff --git a/automation/script/module.py b/automation/script/module.py index e383653d3..64226f574 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2432,8 +2432,34 @@ def test(self, i): logging.info(test_config) variations = meta.get("variations") tags_string = ",".join(meta.get("tags")) + test_input_index = i.get('test_input_index') + test_input_id = i.get('test_input_id') run_inputs = i.get("run_inputs", test_config.get('run_inputs', [ {"docker_os": "ubuntu", "docker_os_version":"22.04"} ])) + if test_input_index: + index_plus = False + try: + if test_input_index.endswith("+"): + input_index = int(test_input_index[:-1]) + index_plus = True + else: + input_index = int(test_input_index) + except ValueError as e: + print(e) + return {'return': 1, 'error': f'Invalid test_input_index: {test_input_index}. Must be an integer or an integer followed by a +'} + if input_index > len(run_inputs): + run_inputs = [] + else: + if index_plus: + run_inputs = run_inputs[index_index-1:] + else: + run_inputs = [ run_inputs[input_index - 1] ] + for run_input in run_inputs: + if test_input_id: + if run_input.get('id', '') != test_input_id: + continue + + ii = {'action': 'run', 'automation':'script', 'quiet': i.get('quiet'), diff --git a/tests/script/process_tests.py b/tests/script/process_tests.py index 99a0ff7ca..a9a7d0e55 100644 --- a/tests/script/process_tests.py +++ b/tests/script/process_tests.py @@ -30,6 +30,8 @@ ii['docker_cm_repo'] = os.environ['DOCKER_CM_REPO'] if os.environ.get('DOCKER_CM_REPO_BRANCH', '') != '': ii['docker_cm_repo_branch'] = os.environ['DOCKER_CM_REPO_BRANCH'] + if os.environ.get('TEST_INPUT_INDEX', '') != '': + ii['test_input_index'] = os.environ['TEST_INPUT_INDEX'] print(ii) r = cm.access(ii)