Merge pull request #12 from ika-rwth-aachen/feature/package-blacklist

Add option to exclude ROS packages from build

.gitlab-ci/docker-ros.yml

@@ -35,6 +35,8 @@ variables:
   ADDITIONAL_FILES_DIR:       docker/additional-files                 # Relative path to directory containing additional files to copy into image"
   ADDITIONAL_PIP_FILE:        docker/additional-pip-requirements.txt  # Relative filepath to file containing additional pip packages to install
   ENABLE_RECURSIVE_ADDITIONAL_PIP:   'false'                          # Enable recursive discovery of files named `additional-pip-file`
+  BLACKLISTED_PACKAGES_FILE:  docker/blacklisted-packages.txt         # Relative filepath to file containing the blacklisted packages
+  ENABLE_RECURSIVE_BLACKLISTED_PACKAGES:  'false'                     # Enable recursive discovery of files named `blacklisted-packages-file`
   CUSTOM_SCRIPT_FILE:         docker/custom.sh                        # Relative filepath to script containing custom installation commands
   ENABLE_RECURSIVE_CUSTOM_SCRIPT:    'false'                          # Enable recursive discovery of files named `custom-script-file`
   VCS_IMPORT_FILE:            .repos                                  # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false)

README.md

@@ -21,6 +21,8 @@
   - [Build multi-arch images on arch-specific self-hosted runners in parallel](#build-multi-arch-images-on-arch-specific-self-hosted-runners-in-parallel)
   - [Build images locally](#build-images-locally)
 - [Advanced Dependencies](#advanced-dependencies)
+  - [Recursion](#recursion)
+  - [Package Blacklist](#package-blacklist)
   - [Extra System Dependencies (*apt*)](#extra-system-dependencies-apt)
   - [Extra System Dependencies (*pip*)](#extra-system-dependencies-pip)
   - [Custom Installation Script](#custom-installation-script)
@@ -38,13 +40,14 @@ We recommend to use *docker-ros* in combination with our other tools for Docker
 
 The Dockerfile performs the following steps to build these images:
 1. All dependency repositories that are defined in a `.repos` file anywhere in the repository are cloned using [*vcstool*](https://github.com/dirk-thomas/vcstool).
-2. The ROS dependencies listed in each package's `package.xml` are installed by [*rosdep*](https://docs.ros.org/en/independent/api/rosdep/html/).
-3. *(optional)* Additional apt dependencies from a special file `additional-debs.txt` are installed, if needed (see [*Advanced Dependencies*](#extra-system-dependencies-apt)).
-4. *(optional)* Additional pip requirements from a special file `additional-pip-requirements.txt` are installed, if needed (see [*Advanced Dependencies*](#extra-system-dependencies-pip)).
-5. *(optional)* A special folder `additional-files/` is copied into the images, if needed (see [*Advanced Dependencies*](#extra-image-files)).
-6. *(optional)* A special script `custom.sh` is executed to perform further arbitrary installation commands, if needed (see [*Advanced Dependencies*](#custom-installation-script)).
-7. *(deployment)* All ROS packages are built using `catkin` (ROS) or `colcon` (ROS2).
-8. *(deployment)* A custom launch command is configured to run on container start.
+2. *(optional)* Packages blacklisted in a special file `blacklisted-packages.txt` are removed from the workspace (see [*Advanced Dependencies*](#package-blacklist)).
+3. The ROS dependencies listed in each package's `package.xml` are installed by [*rosdep*](https://docs.ros.org/en/independent/api/rosdep/html/).
+4. *(optional)* Additional apt dependencies from a special file `additional-debs.txt` are installed, if needed (see [*Advanced Dependencies*](#extra-system-dependencies-apt)).
+5. *(optional)* Additional pip requirements from a special file `additional-pip-requirements.txt` are installed, if needed (see [*Advanced Dependencies*](#extra-system-dependencies-pip)).
+6. *(optional)* A special folder `additional-files/` is copied into the images, if needed (see [*Advanced Dependencies*](#extra-image-files)).
+7. *(optional)* A special script `custom.sh` is executed to perform further arbitrary installation commands, if needed (see [*Advanced Dependencies*](#custom-installation-script)).
+8. *(deployment)* All ROS packages are built using `catkin` (ROS) or `colcon` (ROS2).
+9. *(deployment)* A custom launch command is configured to run on container start.
 
 ### Prerequisites
 
@@ -231,7 +234,7 @@ jobs:
     strategy:
       matrix:
         target: [dev, run]
-        platform: [amd64, arm64]  
+        platform: [amd64, arm64]
     runs-on: [self-hosted, "${{ matrix.platform }}"]
     steps:
       - uses: ika-rwth-aachen/[email protected]
@@ -286,6 +289,16 @@ jobs:
 
 In order to keep things organized, we recommend to place all *docker-ros* related files in a `docker` folder on top repository level.
 
+### Recursion
+
+Most of the steps listed in [*About*](#about) and below can be toggled between recursive and non-recursive mode, see [*Configuration Variables*](#configuration-variables). This usually means that not only special files on the top-level are considered (e.g., `docker/additional-requirements.txt`), but also files with the same name (e.g., `additional-requirements.txt`) that are found anywhere in the workspace, even after cloning the upstream repositories in step 1.
+
+### Package Blacklist
+
+If your ROS-based repository (or any of your repository's upstream dependencies, see `.repos`) contains ROS packages that should neither be built nor be used for determining dependencies, you can blacklist those in a special `blacklisted-packages.txt` file.
+
+Create a file `blacklisted-packages.txt` in your `docker` folder (or configure a different `BLACKLISTED_PACKAGES_FILE`) and list any ROS package name to blacklist.
+
 ### Extra System Dependencies (*apt*)
 
 If your ROS-based repository requires system dependencies that cannot be installed by specifying their [rosdep](https://docs.ros.org/en/independent/api/rosdep/html/) keys in a `package.xml`, you can use a special `additional-debs.txt` file.
@@ -325,12 +338,12 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
 - **`additional-pip-file` | `ADDITIONAL_PIP_FILE`**  
   Relative filepath to file containing additional pip packages to install  
   *default:* `docker/additional-pip-requirements.txt`
-- **`vcs-import-file` | `VCS_IMPORT_FILE`**  
-  Relative filepath to file containing additional repos to install via vcstools (only relevant if `enable-recursive-vcs-import=false`)
-  *default:* `.repos`
 - **`base-image` | `BASE_IMAGE`**  
   Base image `name:tag`  
   *required*  
+- **`blacklisted-packages-file` | `BLACKLISTED_PACKAGES_FILE`**  
+  Relative filepath to file containing blacklisted packages  
+  *default:* `docker/blacklisted-packages.txt`
 - **`build-context` | `BUILD_CONTEXT`**  
   Build context of Docker build process  
   *default:* `${{ github.workspace }}` | `.`  
@@ -373,6 +386,9 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
 - **`enable-recursive-additional-pip` | `ENABLE_RECURSIVE_ADDITIONAL_PIP`**  
   Enable recursive discovery of files named `additional-pip-file`  
   *default:* `false`
+- **`enable-recursive-blacklisted-packages` | `ENABLE_RECURSIVE_BLACKLISTED_PACKAGES`**  
+  Enable recursive discovery of files named `blacklisted-packages-file`  
+  *default:* `false`
 - **`enable-recursive-custom-script` | `ENABLE_RECURSIVE_CUSTOM_SCRIPT`**  
   Enable recursive discovery of files named `custom-script-file`  
   *default:* `false`
@@ -410,3 +426,6 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
   Target stage of Dockerfile (comma-separated list)  
   *default:* `run`
   *supported values:* `dev`, `run`
+- **`vcs-import-file` | `VCS_IMPORT_FILE`**  
+  Relative filepath to file containing additional repos to install via vcstools (only relevant if `enable-recursive-vcs-import=false`)  
+  *default:* `.repos`

action.yml

@@ -88,6 +88,14 @@ inputs:
     description: "Enable recursive discovery of files named `additional-pip-file`"
     default: false
 
+  blacklisted-packages-file:
+    description: "Relative filepath to file containing blacklisted packages to remove from workspace"
+    default: docker/blacklisted-packages.txt
+
+  enable-recursive-blacklisted-packages:
+    description: "Enable recursive discovery of files named `blacklisted-packages-file`"
+    default: false
+
   custom-script-file:
     description: "Relative filepath to script containing custom installation commands"
     default: docker/custom.sh
@@ -172,6 +180,8 @@ runs:
         ADDITIONAL_FILES_DIR: ${{ inputs.additional-files-dir }}
         ADDITIONAL_PIP_FILE: ${{ inputs.additional-pip-file }}
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
+        BLACKLISTED_PACKAGES_FILE: ${{ inputs.blacklisted-packages-file }}
+        ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: ${{ inputs.enable-recursive-blacklisted-packages }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
         VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}
@@ -227,6 +237,8 @@ runs:
         ADDITIONAL_FILES_DIR: ${{ inputs.additional-files-dir }}
         ADDITIONAL_PIP_FILE: ${{ inputs.additional-pip-file }}
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
+        BLACKLISTED_PACKAGES_FILE: ${{ inputs.blacklisted-packages-file }}
+        ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: ${{ inputs.enable-recursive-blacklisted-packages }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
         VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}
@@ -260,6 +272,8 @@ runs:
         ADDITIONAL_FILES_DIR: ${{ inputs.additional-files-dir }}
         ADDITIONAL_PIP_FILE: ${{ inputs.additional-pip-file }}
         ENABLE_RECURSIVE_ADDITIONAL_PIP: ${{ inputs.enable-recursive-additional-pip }}
+        BLACKLISTED_PACKAGES_FILE: ${{ inputs.blacklisted-packages-file }}
+        ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: ${{ inputs.enable-recursive-blacklisted-packages }}
         CUSTOM_SCRIPT_FILE: ${{ inputs.custom-script-file }}
         ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
         VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}

docker/Dockerfile

@@ -63,12 +63,25 @@ RUN apt-get update && \
 
 ARG VCS_IMPORT_FILE=".repos"
 ARG ENABLE_RECURSIVE_VCS_IMPORT="true"
-RUN if [[ $ENABLE_RECURSIVE_ADDITIONAL_DEBS == 'true' ]]; then \
+RUN if [[ $ENABLE_RECURSIVE_VCS_IMPORT == 'true' ]]; then \
         /usr/local/bin/recursive_vcs_import.py src src/upstream ; \
     else \
-        vcs import src/upstream < src/target/${VCS_IMPORT_FILE} ; \
+        [[ -f src/target/${VCS_IMPORT_FILE} ]] && vcs import src/upstream < src/target/${VCS_IMPORT_FILE} ; \
     fi
 
+# remove blacklisted packages from workspace
+ARG BLACKLISTED_PACKAGES_FILE="docker/blacklisted-packages.txt"
+ARG ENABLE_RECURSIVE_BLACKLISTED_PACKAGES="false"
+RUN echo "colcon list -p --base-paths src/ --packages-select \\" >> $WORKSPACE/.remove-packages.sh && \
+    if [[ $ENABLE_RECURSIVE_BLACKLISTED_PACKAGES == 'true' ]]; then \
+        find . -type f -name $(basename ${BLACKLISTED_PACKAGES_FILE}) -exec sed '$a\' {} \; | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.remove-packages.sh ; \
+    elif [[ -f src/target/${BLACKLISTED_PACKAGES_FILE} ]]; then \
+        cat src/target/${BLACKLISTED_PACKAGES_FILE} | awk '{print "  " $0 " \\"}' >> $WORKSPACE/.remove-packages.sh ; \
+    fi && \
+    echo ";" >> $WORKSPACE/.remove-packages.sh && \
+    chmod +x $WORKSPACE/.remove-packages.sh && \
+    $WORKSPACE/.remove-packages.sh 2> /dev/null | xargs rm -rf
+
 # create install script with list of rosdep dependencies
 RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \
     source /opt/ros/$ROS_DISTRO/setup.bash && \

scripts/build.sh

@@ -28,6 +28,8 @@ build_image() {
         $(if [[ -n "${ADDITIONAL_FILES_DIR}" ]]; then echo "--build-arg ADDITIONAL_FILES_DIR=${ADDITIONAL_FILES_DIR}"; fi) \
         $(if [[ -n "${ADDITIONAL_PIP_FILE}" ]]; then echo "--build-arg ADDITIONAL_PIP_FILE=${ADDITIONAL_PIP_FILE}"; fi) \
         $(if [[ -n "${ENABLE_RECURSIVE_ADDITIONAL_PIP}" ]]; then echo "--build-arg ENABLE_RECURSIVE_ADDITIONAL_PIP=${ENABLE_RECURSIVE_ADDITIONAL_PIP}"; fi) \
+        $(if [[ -n "${BLACKLISTED_PACKAGES_FILE}" ]]; then echo "--build-arg BLACKLISTED_PACKAGES_FILE=${BLACKLISTED_PACKAGES_FILE}"; fi) \
+        $(if [[ -n "${ENABLE_RECURSIVE_BLACKLISTED_PACKAGES}" ]]; then echo "--build-arg ENABLE_RECURSIVE_BLACKLISTED_PACKAGES=${ENABLE_RECURSIVE_BLACKLISTED_PACKAGES}"; fi) \
         $(if [[ -n "${CUSTOM_SCRIPT_FILE}" ]]; then echo "--build-arg CUSTOM_SCRIPT_FILE=${CUSTOM_SCRIPT_FILE}"; fi) \
         $(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \
         $(if [[ -n "${VCS_IMPORT_FILE}" ]]; then echo "--build-arg VCS_IMPORT_FILE=${VCS_IMPORT_FILE}"; fi) \

scripts/ci.sh

@@ -31,6 +31,8 @@ ENABLE_RECURSIVE_ADDITIONAL_DEBS="${ENABLE_RECURSIVE_ADDITIONAL_DEBS:-}"
 ADDITIONAL_FILES_DIR="${ADDITIONAL_FILES_DIR:-}"
 ADDITIONAL_PIP_FILE="${ADDITIONAL_PIP_FILE:-}"
 ENABLE_RECURSIVE_ADDITIONAL_PIP="${ENABLE_RECURSIVE_ADDITIONAL_PIP:-}"
+BLACKLISTED_PACKAGES_FILE="${BLACKLISTED_PACKAGES_FILE:-}"
+ENABLE_RECURSIVE_BLACKLISTED_PACKAGES="${ENABLE_RECURSIVE_BLACKLISTED_PACKAGES:-}"
 CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}"
 ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}"
 VCS_IMPORT_FILE="${VCS_IMPORT_FILE:-}"

templates/docker-compose.template.yml

@@ -14,10 +14,12 @@ x-build: &build
     ADDITIONAL_DEBS_FILE: $ADDITIONAL_DEBS_FILE
     ADDITIONAL_FILES_DIR: $ADDITIONAL_FILES_DIR
     ADDITIONAL_PIP_FILE: $ADDITIONAL_PIP_FILE
+    BLACKLISTED_PACKAGES_FILE: $BLACKLISTED_PACKAGES_FILE
     VCS_IMPORT_FILE: $VCS_IMPORT_FILE
     CUSTOM_SCRIPT_FILE: $CUSTOM_SCRIPT_FILE
     ENABLE_RECURSIVE_ADDITIONAL_DEBS: $ENABLE_RECURSIVE_ADDITIONAL_DEBS
     ENABLE_RECURSIVE_ADDITIONAL_PIP: $ENABLE_RECURSIVE_ADDITIONAL_PIP
+    ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: $ENABLE_RECURSIVE_BLACKLISTED_PACKAGES
     ENABLE_RECURSIVE_CUSTOM_SCRIPT: $ENABLE_RECURSIVE_CUSTOM_SCRIPT
     ENABLE_RECURSIVE_VCS_IMPORT: $ENABLE_RECURSIVE_VCS_IMPORT
     GIT_HTTPS_PASSWORD: $GIT_HTTPS_PASSWORD