Sync Docs (#1908)

Docs update

docs/benchmarks/language/reproducibility/indyscc24-bert.md

@@ -37,7 +37,6 @@ All the needed files are automatically pushed to the GitHub repository if you ma
 ```bash
 cm run script --tags=generate,inference,submission \
    --clean \
-   --preprocess_submission=yes \
    --run-checker \
    --tar=yes \
    --env.CM_TAR_OUTFILE=submission.tar.gz \

docs/benchmarks/text_to_image/reproducibility/scc24.md

@@ -59,10 +59,10 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt
 
 ### Generate actual submission tree
 
+
 ```bash
 cm run script --tags=generate,inference,submission \
    --clean \
-   --preprocess_submission=yes \
    --run-checker \
    --tar=yes \
    --env.CM_TAR_OUTFILE=submission.tar.gz \

docs/submission/index.md

@@ -2,7 +2,16 @@
 hide:
   - toc
 ---
-[![Streamline your MLPerf results using CM Framework](https://img.youtube.com/vi/eI1Hoecc3ho/0.jpg)](https://youtu.be/eI1Hoecc3ho)
+
+<p align="center">
+  <img src="../img/submission-flow.png" alt="Submission Generation Flow">
+</p>
+
+<p align="center"><em>Figure: MLPerf Inference Submission Generation Flow</em></p>
+
+<!--![Submission Generation Flow](../img/submission-flow.png)-->
+
+Click [here](https://youtu.be/eI1Hoecc3ho) to view the recording of the workshop: Streamlining your MLPerf Inference results using CM.
 
 === "CM based benchmark"
     If you have followed the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, all the valid results will get aggregated to the `cm cache` folder. The following command could be used to browse the structure of inference results folder generated by CM.

main.py

@@ -122,6 +122,7 @@ def mlperf_inference_implementation_readme(
         if not categories:
             if model.lower() == "bert-99.9":
                 categories = ["Datacenter"]
+
             elif (
                 "dlrm" in model.lower()
                 or "llama2" in model.lower()
@@ -148,7 +149,7 @@ def mlperf_inference_implementation_readme(
                 scenarios = [
                     scenario for scenario in scenarios if scenario in fixed_scenarios]
 
-            content += f'{pre_space}=== "{category.lower()}"\n\n'
+            content += f"{pre_space}=== \"{category.lower()}\"\n\n"
 
             cur_space = pre_space + "    "
             scenarios_string = ", ".join(scenarios)
@@ -173,6 +174,7 @@ def mlperf_inference_implementation_readme(
 
                     # minimum system requirements
                     content += get_min_system_requirements(
+
                         cur_space2, model, implementation, device
                     )
 
@@ -235,13 +237,20 @@ def mlperf_inference_implementation_readme(
                                 extra_docker_input_string,
                             )
 
+                            common_info = get_common_info(
+                                spaces + 16,
+                                implementation
+                            )
+
                             if (
                                 execution_env == "Native"
                             ):  # Native implementation steps through virtual environment
                                 content += f"{cur_space3}####### Setup a virtual environment for Python\n"
                                 content += get_venv_command(spaces + 16)
                                 content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n"
 
+                                content += common_info
+
                                 content += setup_run_cmd.replace(
                                     "--docker ", "")
 
@@ -256,6 +265,9 @@ def mlperf_inference_implementation_readme(
                                     device,
                                     setup_tips,
                                 )
+
+                                content += common_info
+
                                 content += docker_info
 
                                 content += setup_run_cmd
@@ -373,7 +385,8 @@ def mlperf_inference_implementation_readme(
                                 extra_input_string,
                             )
                             content += run_cmd
-                            content += run_suffix
+
+                        content += run_suffix
 
         readme_prefix = get_readme_prefix(
             spaces, model, implementation, extra_variation_tags
@@ -473,6 +486,24 @@ def get_venv_command(spaces):
 {pre_space}export CM_SCRIPT_EXTRA_CMD=\"--adr.python.name=mlperf\"
 {pre_space}```\n"""
 
+    # contains run command information which is common to both docker and
+    # native runs
+    def get_common_info(spaces, implementation):
+        info = ""
+        pre_space = ""
+        for i in range(1, spaces):
+            pre_space = pre_space + " "
+        pre_space += " "
+        # pre_space = "                "
+        info += f"\n{pre_space}!!! tip\n\n"
+        info += f"{pre_space}    - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n"
+        if implementation.lower() == "reference":
+            info += f"{pre_space}    - Add `--adr.mlperf-implementation.tags=_branch.master,_repo.<CUSTOM_INFERENCE_REPO_LINK>` if you are modifying the official MLPerf Inference implementation in a custom fork.\n\n"
+            info += f"{pre_space}    - Add `--adr.inference-src.tags=_repo.<CUSTOM_INFERENCE_REPO_LINK>` if you are modifying the model config accuracy script in the submission checker within a custom fork.\n\n"
+            info += f"{pre_space}    - Add `--adr.inference-src.version=custom` if you are using the modified MLPerf Inference code or accuracy script on submission checker within a custom fork.\n\n"
+
+        return info
+
     def get_docker_info(spaces, model, implementation,
                         device, setup_tips=True):
         info = ""
@@ -487,7 +518,6 @@ def get_docker_info(spaces, model, implementation,
             if model == "sdxl":
                 info += f"{pre_space}    - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n"
 
-            info += f"{pre_space}    - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n"
             if implementation.lower() == "nvidia":
                 info += f"{pre_space}    - Default batch size is assigned based on [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n"
                 info += f"{pre_space}    - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n"
@@ -499,6 +529,10 @@ def get_docker_info(spaces, model, implementation,
                 info += f"\n{pre_space}!!! tip\n\n"
                 info += f"{pre_space}    - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n"
 
+        # return empty string if nothing is filled inside the tip
+        if info == f"\n{pre_space}!!! tip\n\n":
+            return ""
+
         return info
 
     def get_readme_prefix(spaces, model, implementation, extra_variation_tags):

tools/submission/generate_final_report.py

@@ -201,6 +201,7 @@ def MakeWorksheet(df, index, filter_dict, sheet_name, outjsondata=[]):
             df = df[value(df[key])]
         if df.size == 0:
             return
+
         json_df = df.to_json(orient="records")
         outjsondata += json.loads(json_df)
 
@@ -328,6 +329,7 @@ def reformatlink(data, key):
                    for key in keystomatch):
                 # print(result)
                 # print(outjsondata[i+1])
+
                 if "Watts" in result["Units"] or "joules" in result["Units"]:
                     result["Performance_Result"] = outjsondata[i + 1]["Result"]
                     result["Performance_Units"] = outjsondata[i + 1]["Units"]