Embed mlperf.conf as a binary string in the distribution

.github/workflows/build_wheels.yml

@@ -80,7 +80,7 @@ jobs:
         run: python -m pip install cibuildwheel twine
 
       - name: Build wheels
-        run: python -m cibuildwheel loadgen/ --output-dir wheels
+        run: git pull && python -m cibuildwheel loadgen/ --output-dir wheels
 
        # Save wheels as artifacts
       - name: Upload built wheels

loadgen/CMakeLists.txt

@@ -38,6 +38,9 @@ message(STATUS "Using output path: ${LIBRARY_OUTPUT_PATH}")
 find_package(PythonInterp)
 message(STATUS "Using Python interpreter: ${PYTHON_EXECUTABLE}")
 
+# Generate source file with version info.
+execute_process(COMMAND xdd -i ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf > ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h)
+
 # Generate source file with version info.
 execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/version_generator.py ${CMAKE_BINARY_DIR}/version_generated.cc ${CMAKE_CURRENT_SOURCE_DIR})
 
@@ -58,6 +61,7 @@ set(SOURCE
   ${CMAKE_CURRENT_SOURCE_DIR}/results.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/version.cc
   ${CMAKE_CURRENT_SOURCE_DIR}/version.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/mlperf_conf.h
   ${CMAKE_BINARY_DIR}/version_generated.cc
 )
 
@@ -82,6 +86,3 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
 	DESTINATION ${CMAKE_INSTALL_PREFIX}/include FILES_MATCHING PATTERN "*.h")
 install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mlperf.conf
         DESTINATION ${CMAKE_INSTALL_PREFIX}/include)
-
-# Define preprocessor macro with the path to mlperf.conf
-add_definitions(-DMLPERF_CONF_PATH=\"${CMAKE_INSTALL_PREFIX}/include/mlperf.conf\")

loadgen/VERSION.txt

@@ -1 +1 @@
-4.1.7
+4.1.9

loadgen/setup.py

@@ -29,6 +29,7 @@
 from pybind11 import get_include
 from pybind11.setup_helpers import Pybind11Extension, build_ext
 from version_generator import generate_loadgen_version_definitions
+import subprocess
 
 generated_version_source_filename = "generated/version_generated.cc"
 generate_loadgen_version_definitions(generated_version_source_filename, ".")
@@ -41,7 +42,7 @@
     "test_settings.h",
     "issue_query_controller.h",
     "early_stopping.h",
-    "query_dispatch_library.h",
+    "query_dispatch_library.h"
 ]
 
 lib_headers = [
@@ -52,7 +53,8 @@
     "version.h",
     "results.h",
     "bindings/c_api.h",
-    "version_generator.py"
+    "version_generator.py",
+    "mlperf_conf.h"
 ]
 
 lib_sources = [
@@ -88,8 +90,14 @@
 version_split = version.split(".")
 
 if len(version_split) < 2:
-    print("Version is incomplete. Needs a format like 4.1 in VERSION file")
+    print("Version is incomplete. Needs a format like 4.1.1 in VERSION file")
 
+command = ["xxd", "-i", "mlperf.conf", "mlperf_conf.h" ]
+try:
+    subprocess.check_call(command)
+except subprocess.CalledProcessError as e:
+    print(f"Failed to generate the mlperf_conf.h file: {e}")
+    raise
 
 mlperf_loadgen_module = Pybind11Extension(
     "mlperf_loadgen",

loadgen/test_settings_internal.cc

@@ -11,6 +11,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "test_settings_internal.h"
+#include "mlperf_conf.h"
 
 #include <fstream>
 #include <map>
@@ -527,7 +528,7 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model,
     if (configCount == 0) {
       // Only allow userConf as the single configFile and loadgen loads the
       // mlperfConf automatically
-      FromConfig(MLPERF_CONF_PATH, model, scenario, true);
+      FromConfig(NULL, model, scenario, true);
     }
 
     else {
@@ -579,24 +580,34 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model,
     return true;
   };
 
-  // dirt simple config parser
-  std::ifstream fss(path);
-  std::string line;
   int line_nr = 0;
   int errors = 0;
-  if (!fss.is_open()) {
-    LogDetail([p = path](AsyncDetail &detail) {
+  // Declare the input stream before the if-else block
+  std::unique_ptr<std::istream> fss;
+  std::string line;
+
+  if (!is_mlperf_conf) {
+    // dirt simple config parser
+    fss = std::make_unique<std::ifstream>(path);
+    if (!static_cast<std::ifstream*>(fss.get())->is_open()) {
+      LogDetail([p = path](AsyncDetail &detail) {
 #if USE_NEW_LOGGING_FORMAT
-      std::stringstream ss;
-      ss << "can't open file " << p;
-      MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str());
+        std::stringstream ss;
+        ss << "can't open file " << p;
+        MLPERF_LOG_ERROR(detail, "error_invalid_config", ss.str());
 #else
-      detail.Error("can't open file ", p);
+        detail.Error("can't open file ", p);
 #endif
-    });
-    return -ENOENT;
+      });
+      return -ENOENT;
+    }
+  } else {
+    // Convert unsigned char array to std::string
+    std::string config_str(reinterpret_cast<const char*>(mlperf_conf), mlperf_conf_len);
+    fss = std::make_unique<std::istringstream>(config_str);
+
   }
-  while (std::getline(fss, line)) {
+  while (std::getline(*fss, line)) {
     line_nr++;
     std::istringstream iss(line);
     std::string s, k;