ci(json): Add configuration requirements to ci.json files (#10385)

* ci(json): Add support for checking sdkconfig before running tests * docs(ci): Add explanation about requires field in JSON * fix(json): Ignore comments when searching requirements * feat(json): Add extended regex support to requires field * change(json): Move to using requirements in JSON * fix(json): Fix requirements for touch tests * refactor(json): Fix formatting of JSON files * fix(spi): Fix SPI example and JSON
espressif · Sep 30, 2024 · e403f0b · e403f0b
1 parent 1f1de27
commit e403f0b
Show file tree

Hide file tree

Showing 169 changed files with 654 additions and 688 deletions.
diff --git a/.github/scripts/install-platformio-esp32.sh b/.github/scripts/install-platformio-esp32.sh
@@ -6,6 +6,7 @@ PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git"
 TOOLCHAIN_VERSION="12.2.0+20230208"
 ESPTOOLPY_VERSION="~1.40501.0"
 ESPRESSIF_ORGANIZATION_NAME="espressif"
+LIBS_DIR="tools/esp32-arduino-libs"
 
 echo "Installing Python Wheel ..."
 pip install wheel > /dev/null 2>&1
@@ -88,12 +89,25 @@ function count_sketches(){ # count_sketches <examples-path>
         local sketchname=$(basename $sketch)
         if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
             continue
+        elif [ -f $sketchdir/ci.json ]; then
+            # If the target is listed as false, skip the sketch. Otherwise, include it.
+            is_target=$(jq -r '.targets[esp32]' $sketchdir/ci.json)
+            if [[ "$is_target" == "false" ]]; then
+                continue
+            fi
+
+            # Check if the sketch requires any configuration options
+            requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
+            if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+                for requirement in $requirements; do
+                    found_line=$(grep -E "^$requirement" $LIBS_DIR/esp32/sdkconfig)
+                    if [[ "$found_line" == "" ]]; then
+                        continue 2
+                    fi
+                done
+            fi
         fi
-        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
-        # If the target is listed as false, skip the sketch. Otherwise, include it.
-        if [[ "$is_target" == "false" ]]; then
-            continue
-        fi
+
         echo $sketch >> sketches.txt
         sketchnum=$(($sketchnum + 1))
     done
@@ -163,12 +177,27 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
-        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
-        # If the target is listed as false, skip the sketch. Otherwise, include it.
-        if [ "${sketchdirname}.ino" != "$sketchname" ] \
-        || [[ "$is_target" == "false" ]]; then
+        if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
             continue
+        elif [ -f $sketchdir/ci.json ]; then
+            # If the target is listed as false, skip the sketch. Otherwise, include it.
+            is_target=$(jq -r '.targets[esp32]' $sketchdir/ci.json)
+            if [[ "$is_target" == "false" ]]; then
+                continue
+            fi
+
+            # Check if the sketch requires any configuration options
+            requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
+            if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+                for requirement in $requirements; do
+                    found_line=$(grep -E "^$requirement" $LIBS_DIR/esp32/sdkconfig)
+                    if [[ "$found_line" == "" ]]; then
+                        continue 2
+                    fi
+                done
+            fi
         fi
+
         sketchnum=$(($sketchnum + 1))
         if [ "$sketchnum" -le "$start_index" ] \
         || [ "$sketchnum" -gt "$end_index" ]; then

diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+LIBS_DIR="tools/esp32-arduino-libs"
+
 function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [extra-options]
     while [ ! -z "$1" ]; do
         case "$1" in
@@ -140,16 +142,25 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
 
     sketchname=$(basename $sketchdir)
 
-    # If the target is listed as false, skip the sketch. Otherwise, include it.
     if [ -f $sketchdir/ci.json ]; then
+        # If the target is listed as false, skip the sketch. Otherwise, include it.
         is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
-    else
-        is_target="true"
-    fi
+        if [[ "$is_target" == "false" ]]; then
+            echo "Skipping $sketchname for target $target"
+            exit 0
+        fi
 
-    if [[ "$is_target" == "false" ]]; then
-        echo "Skipping $sketchname for target $target"
-        exit 0
+        # Check if the sketch requires any configuration options
+        requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
+        if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+            for requirement in $requirements; do
+                found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
+                if [[ "$found_line" == "" ]]; then
+                    echo "Target $target does not meet the requirement $requirement for $sketchname. Skipping."
+                    exit 0
+                fi
+            done
+        fi
     fi
 
     ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
@@ -288,16 +299,23 @@ function count_sketches(){ # count_sketches <path> [target] [file]
         local sketchname=$(basename $sketch)
         if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
             continue
-        elif [[ -n $target ]]; then
+        elif [[ -n $target ]] && [[ -f $sketchdir/ci.json ]]; then
             # If the target is listed as false, skip the sketch. Otherwise, include it.
-            if [ -f $sketchdir/ci.json ]; then
-                is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
-            else
-                is_target="true"
-            fi
+            is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
             if [[ "$is_target" == "false" ]]; then
                 continue
             fi
+
+            # Check if the sketch requires any configuration options
+            requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
+            if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+                for requirement in $requirements; do
+                    found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
+                    if [[ "$found_line" == "" ]]; then
+                        continue 2
+                    fi
+                done
+            fi
         fi
         echo $sketch >> sketches.txt
         sketchnum=$(($sketchnum + 1))

diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh
@@ -10,19 +10,29 @@ function run_test() {
     local result=0
     local error=0
 
-    # If the target or platform is listed as false, skip the sketch. Otherwise, include it.
     if [ -f $sketchdir/ci.json ]; then
+        # If the target or platform is listed as false, skip the sketch. Otherwise, include it.
         is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
         selected_platform=$(jq -r --arg platform $platform '.platforms[$platform]' $sketchdir/ci.json)
-    else
-        is_target="true"
-        selected_platform="true"
-    fi
 
-    if [[ $is_target == "false" ]] || [[ $selected_platform == "false" ]]; then
-      printf "\033[93mSkipping $sketchname test for $target, platform: $platform\033[0m\n"
-      printf "\n\n\n"
-      return 0
+        if [[ $is_target == "false" ]] || [[ $selected_platform == "false" ]]; then
+            printf "\033[93mSkipping $sketchname test for $target, platform: $platform\033[0m\n"
+            printf "\n\n\n"
+            return 0
+        fi
+
+        # Check if the sketch requires any configuration options
+        requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
+        if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+            for requirement in $requirements; do
+                found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
+                if [[ "$found_line" == "" ]]; then
+                    printf "\033[93mTarget $target does not meet the requirement $requirement for $sketchname. Skipping.\033[0m\n"
+                    printf "\n\n\n"
+                    return 0
+                fi
+            done
+        fi
     fi
 
     if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then
@@ -110,6 +120,7 @@ function run_test() {
 
 SCRIPTS_DIR="./.github/scripts"
 COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
+LIBS_DIR="tools/esp32-arduino-libs"
 
 platform="hardware"
 wokwi_timeout=60000

diff --git a/docs/en/contributing.rst b/docs/en/contributing.rst
@@ -109,17 +109,44 @@ Also:
 Testing
 *******
 
-Be sure you have tested the example in all the supported targets. If the example works only with specific targets,
-edit/add the ``ci.json`` in the same folder as the sketch to specify the supported targets. By default,
-all targets are assumed to be supported.
+Be sure you have tested the example in all the supported targets. If the example some specific hardware requirements,
+edit/add the ``ci.json`` in the same folder as the sketch to specify the regular expression for the
+required configurations from ``sdkconfig``.
+This will ensure that the CI system will run the test only on the targets that have the required configurations.
 
-Here is an example of the ``ci.json`` file where the example does not support ESP32-H2 and ESP32-S2:
+You can check the available configurations in the ``sdkconfig`` file in the ``tools/esp32-arduino-libs/<target>`` folder.
+
+Here is an example of the ``ci.json`` file where the example requires Wi-Fi to work properly:
+
+.. code-block:: json
+
+    {
+      "requires": [
+        "CONFIG_SOC_WIFI_SUPPORTED=y"
+      ]
+    }
+
+.. note::
+
+    The list of configurations will be checked against the ``sdkconfig`` file in the target folder. If the configuration is not present in the ``sdkconfig``,
+    the test will be skipped for that target. That means that the test will only run on the targets that have **ALL** the required configurations.
+
+    Also, by default, the "match start of line" character (``^``) will be added to the beginning of each configuration.
+    That means that the configuration must be at the beginning of the line in the ``sdkconfig`` file.
+
+Sometimes, the example might not be supported by some target, even if the target has the required configurations
+(like resources limitations or requiring a specific SoC). To avoid compilation errors, you can add the target to the ``ci.json``
+file so the CI system will force to skip the test on that target.
+
+Here is an example of the ``ci.json`` file where the example is requires Wi-Fi to work properly but is also not supported by the ESP32-S2 target:
 
 .. code-block:: json
 
     {
+      "requires": [
+        "CONFIG_SOC_WIFI_SUPPORTED=y"
+      ],
       "targets": {
-        "esp32h2": false,
         "esp32s2": false
       }
     }
@@ -130,17 +157,17 @@ For example, in the sketch:
 .. code-block:: arduino
 
     /*
-      THIS FEATURE IS SUPPORTED ONLY BY ESP32-S2 AND ESP32-C3
+      THIS FEATURE REQUIRES WI-FI SUPPORT AND IS NOT AVAILABLE FOR ESP32-S2 AS IT DOES NOT HAVE ENOUGH RAM.
     */
 
 And in the ``README.md`` file:
 
 .. code-block:: markdown
 
-    Currently, this example supports the following targets.
+    Currently, this example requires Wi-Fi and supports the following targets.
 
-    | Supported Targets | ESP32 | ESP32-S2 | ESP32-C3 | ESP32-S3 |
-    | ----------------- | ----- | -------- | -------- | -------- |
+    | Supported Targets | ESP32 | ESP32-H2 | ESP32-S3 | ESP32-C3 | ESP32-C6 |
+    | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 Example Template
 ****************
@@ -341,8 +368,11 @@ CI JSON File
 
 The ``ci.json`` file is used to specify how the test suite and sketches will handled by the CI system. It can contain the following fields:
 
-* ``targets``: A dictionary that specifies the supported targets. The key is the target name and the value is a boolean that specifies if the
-  target is supported. By default, all targets are assumed to be supported. This field is also valid for examples.
+* ``requires``: A list of configurations in ``sdkconfig`` that are required to run the test suite. The test suite will only run on the targets
+  that have the required configurations. By default, no configurations are required.
+* ``targets``: A dictionary that specifies the targets for which the tests will be run. The key is the target name and the value is a boolean
+  that specifies if the test should be run for that target. By default, all targets are enabled as long as they have the required configurations
+  specified in the ``requires`` field. This field is also valid for examples.
 * ``platforms``: A dictionary that specifies the supported platforms. The key is the platform name and the value is a boolean that specifies if
   the platform is supported. By default, all platforms are assumed to be supported.
 * ``extra_tags``: A list of extra tags that the runner will require when running the test suite in hardware. By default, no extra tags are required.

diff --git a/libraries/ArduinoOTA/examples/BasicOTA/ci.json b/libraries/ArduinoOTA/examples/BasicOTA/ci.json
@@ -1,5 +1,5 @@
 {
-  "targets": {
-    "esp32h2": false
-  }
+  "requires": [
+    "CONFIG_SOC_WIFI_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/AsyncUDP/examples/AsyncUDPClient/ci.json b/libraries/AsyncUDP/examples/AsyncUDPClient/ci.json
@@ -1,5 +1,5 @@
 {
-  "targets": {
-    "esp32h2": false
-  }
+  "requires": [
+    "CONFIG_SOC_WIFI_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/AsyncUDP/examples/AsyncUDPMulticastServer/ci.json b/libraries/AsyncUDP/examples/AsyncUDPMulticastServer/ci.json
@@ -1,5 +1,5 @@
 {
-  "targets": {
-    "esp32h2": false
-  }
+  "requires": [
+    "CONFIG_SOC_WIFI_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/AsyncUDP/examples/AsyncUDPServer/ci.json b/libraries/AsyncUDP/examples/AsyncUDPServer/ci.json
@@ -1,5 +1,5 @@
 {
-  "targets": {
-    "esp32h2": false
-  }
+  "requires": [
+    "CONFIG_SOC_WIFI_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/BLE5_extended_scan/ci.json b/libraries/BLE/examples/BLE5_extended_scan/ci.json
@@ -1,6 +1,5 @@
 {
-  "targets": {
-    "esp32": false,
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_50_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/BLE5_multi_advertising/ci.json b/libraries/BLE/examples/BLE5_multi_advertising/ci.json
@@ -1,6 +1,5 @@
 {
-  "targets": {
-    "esp32": false,
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_50_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/BLE5_periodic_advertising/ci.json b/libraries/BLE/examples/BLE5_periodic_advertising/ci.json
@@ -1,6 +1,5 @@
 {
-  "targets": {
-    "esp32": false,
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_50_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/BLE5_periodic_sync/ci.json b/libraries/BLE/examples/BLE5_periodic_sync/ci.json
@@ -1,6 +1,5 @@
 {
-  "targets": {
-    "esp32": false,
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_50_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/Beacon_Scanner/ci.json b/libraries/BLE/examples/Beacon_Scanner/ci.json
@@ -1,5 +1,5 @@
 {
-  "targets": {
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/Client/ci.json b/libraries/BLE/examples/Client/ci.json
@@ -1,5 +1,5 @@
 {
-  "targets": {
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/EddystoneTLM_Beacon/ci.json b/libraries/BLE/examples/EddystoneTLM_Beacon/ci.json
@@ -1,6 +1,5 @@
 {
-  "targets": {
-    "esp32h2": false,
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/EddystoneURL_Beacon/ci.json b/libraries/BLE/examples/EddystoneURL_Beacon/ci.json
@@ -1,6 +1,5 @@
 {
-  "targets": {
-    "esp32h2": false,
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_SUPPORTED=y"
+  ]
 }
diff --git a/libraries/BLE/examples/Notify/ci.json b/libraries/BLE/examples/Notify/ci.json
@@ -1,5 +1,5 @@
 {
-  "targets": {
-    "esp32s2": false
-  }
+  "requires": [
+    "CONFIG_SOC_BLE_SUPPORTED=y"
+  ]
 }