F#src 7323 tidy up ros2 base images

ansible/callback_plugins/log_beautifier.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import
 import json
 import re
+import sys
 from threading import Timer
 from ansible.plugins.callback import CallbackBase
 
@@ -15,7 +16,10 @@ def obfuscate_credentials(input_value):
 
 def fixed_dump_results(self, result, indent=None, sort_keys=True, keep_invocation=False):
     json_message = self._original_dump_results(result, indent, sort_keys, keep_invocation)
-    message_dictionary = json.loads(json_message, encoding="utf-8")
+    if sys.version_info.major>=3 and sys.version_info.minor>=9:
+        message_dictionary = json.loads(json_message)
+    else:
+        message_dictionary = json.loads(json_message, encoding="utf-8")
     result = ""
     for key, value in message_dictionary.items():
         if key not in ["stderr", "stdout_lines"]:

ansible/data/jammy/requirements.txt

@@ -0,0 +1,20 @@
+ansible
+ansible-core
+asn1crypto
+bcrypt
+cffi
+cryptography
+ipaddress
+Jinja2
+MarkupSafe
+packaging
+paramiko
+pbr
+pyasn1
+pycparser
+pycrypto
+PyNaCl
+pyopenssl
+resolvelib
+six
+testresources

ansible/deploy.sh

@@ -203,7 +203,8 @@ echo " |   Setting up requirements      |"
 echo " ----------------------------------"
 echo ""
 
-sudo pip3 install -r "${SR_BUILD_TOOLS_HOME}/ansible/data/requirements.txt"
+export ubuntu_version="$(lsb_release -cs)"
+sudo pip3 install -r "${SR_BUILD_TOOLS_HOME}/ansible/data/$ubuntu_version/requirements.txt"
 sudo pip3 install --upgrade setuptools==51.1.1
 
 echo ""

ansible/docker_site.yml

@@ -35,13 +35,13 @@
                                                                  "xml_code_style_check", "catkin_code_style_check",
                                                                  "code_style_check", "create_workspace",
                                                                  "benchmarking", "check_install"]}
-    - {role: build_deps/ur-client-library, tags: ["build", "unit_tests", "software_tests", "all_tests",
-                                                  "code_coverage", "codecov_tool", "python_code_coverage",
-                                                  "cpp_code_coverage", "check_deb_make", "complete_deb_check",
-                                                  "check_build", "python_code_style_check",
-                                                  "cpp_code_style_check", "xml_code_style_check",
-                                                  "catkin_code_style_check", "code_style_check",
-                                                  "create_workspace", "benchmarking", "check_install"]}
+    - {role: build_deps/ur-client-library, when: (ros_release == "noetic"), tags: ["build", "unit_tests", "software_tests", "all_tests",
+                                                                                   "code_coverage", "codecov_tool", "python_code_coverage",
+                                                                                   "cpp_code_coverage", "check_deb_make", "complete_deb_check",
+                                                                                   "check_build", "python_code_style_check",
+                                                                                   "cpp_code_style_check", "xml_code_style_check",
+                                                                                   "catkin_code_style_check", "code_style_check",
+                                                                                   "create_workspace", "benchmarking", "check_install"]}
     - {role: ci/update_dependencies, when: run_build, tags: ["build", "unit_tests", "software_tests", "all_tests",
                                                              "code_coverage", "codecov_tool", "python_code_coverage",
                                                              "cpp_code_coverage", "check_deb_make",

ansible/roles/ci/init/create_workspace/tasks/main.yml

@@ -61,6 +61,4 @@
  #       chdir: "/home/user/workspace/"
  #     become: yes
  #     become_method: sudo
-  when:
-    - ros_release == 'rolling' 
-    - ros_release == 'humble'
+  when: (ros_release == 'rolling') or (ros_release == 'humble')

ansible/roles/ci/init/install/tasks/main.yml

@@ -18,17 +18,18 @@
     - ros_release != 'rolling' 
     - ros_release != 'humble'
 
+# Using a specific version for empy as that's the one that is installed by default in Ubuntu Jammy from debian repos
+# so pip will skip it. Otherwise pip throws an error when failing to uninstall the deb version to install the latest pip version.
+# I don't really know what this package is needed for.
 - name: Install PIP modules
   pip: 
-    name: ['catkin_pkg','empy','coverage','catkin-lint','pylint']
+    name: ['catkin_pkg','empy==3.3.4','coverage','catkin-lint','pylint']
     extra_args: '--upgrade'
     executable: pip3
   become: yes
   become_method: sudo
   become_user: root
-  when: 
-    - ros_release == 'rolling' 
-    - ros_release == 'humble'
+  when: (ros_release == 'rolling') or (ros_release == 'humble')
 
 - name: Install PIP modules
   pip: 

bin/sr-run-ci-build.sh

@@ -24,7 +24,13 @@ if  [ "semaphore_docker" != $server_type ] && [ "local" != $server_type ] && [ "
 
   sudo apt-get install -y python3-dev libxml2-dev libxslt-dev python3-pip lcov wget git libssl-dev libffi-dev libyaml-dev
   sudo pip3 install --upgrade pip setuptools==51.1.1 gcovr
-  sudo pip3 install PyYAML==5.4.1 --ignore-installed
+  if [ "$ubuntu_version" != "jammy" ]
+  then
+    sudo pip3 install PyYAML==5.4.1 --ignore-installed
+  else
+    sudo pip3 install "PyYAML>=5.4.1" --ignore-installed
+  fi
+
 
   git config --global user.email "[email protected]"
   git config --global user.name "Build Tools"
@@ -34,7 +40,7 @@ if  [ "semaphore_docker" != $server_type ] && [ "local" != $server_type ] && [ "
   git clone --depth 1 https://github.com/shadow-robot/sr-build-tools.git -b "$toolset_branch" $build_tools_folder
   cd $build_tools_folder/ansible
 
-  sudo pip3 install -r data/requirements.txt
+  sudo pip3 install -r data/$ubuntu_version/requirements.txt
 fi
 
 export extra_variables="codecov_secure=$CODECOV_TOKEN github_login=$GITHUB_LOGIN github_password=$GITHUB_PASSWORD ros_release=$ros_release ubuntu_version_name=$ubuntu_version "
@@ -83,7 +89,7 @@ case $server_type in
   ;;
 
 "docker_hub") echo "Docker Hub"
-  PYTHONUNBUFFERED=1 ansible-playbook -v -i "localhost," -c local docker_site.yml --tags "docker_hub,$tags_list" -e "ros_release=$ros_release ubuntu_version_name=$ubuntu_version"
+  PYTHONUNBUFFERED=1 ansible-playbook -vvv -i "localhost," -c local docker_site.yml --tags "docker_hub,$tags_list" -e "ros_release=$ros_release ubuntu_version_name=$ubuntu_version"
   ;;
 
 "local") echo "Local run"

docker/ros2/humble/Dockerfile

@@ -1,5 +1,5 @@
 #
-# ROS2 rolling with build tools Dockerfile
+# ROS2 humble with build tools Dockerfile
 #
 # https://github.com/shadow-robot/sr-build-tools/
 #
@@ -18,9 +18,10 @@ ENV toolset_branch ${toolset_branch:-"master"}
 ENV server_type="docker_hub"
 ENV used_modules="check_cache,create_workspace"
 ENV ros_release_name=humble
+ENV ubuntu_version_name=jammy
 ENV remote_shell_script="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/bin/sr-run-ci-build.sh"
 
-ENV remote_additional_bashrc_cmds="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/docker/utils/additional_bashrc_cmds_rolling"
+ENV remote_additional_bashrc_cmds="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/docker/utils/additional_bashrc_cmds_humble"
 
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 COPY terminator_config /tmp/terminator_config

docker/ros2/humble/buildspec.yml

@@ -5,17 +5,14 @@ env:
   variables:
     branch: default
     image: build-tools
-    tag: focal-humble
+    tag: humble
     dockerfile_folder: docker/ros2/humble/
   exported-variables:
     - branch
     - image
     - tag
     - dockerfile_folder
 phases:
-  install:
-    runtime-versions:
-      docker: 19
   build:
     commands:
       - |
@@ -29,15 +26,10 @@ phases:
         else
           branch=$CODEBUILD_SOURCE_VERSION
         fi
-      - awscli_v1_executable=$(which aws)
-      - rm $awscli_v1_executable
-      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-      - unzip awscliv2.zip
-      - ./aws/install
       - git checkout $branch
       - cd $dockerfile_folder
       - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot
-      - docker build --build-arg toolset_branch=$branch-t $image .
+      - docker build --build-arg toolset_branch=$branch -t $image .
       - docker tag $image:latest public.ecr.aws/shadowrobot/$image:$tag
       - docker push public.ecr.aws/shadowrobot/$image:$tag
       - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt

docker/ros2/rolling/Dockerfile

@@ -4,7 +4,7 @@
 # https://github.com/shadow-robot/sr-build-tools/
 #
 
-FROM osrf/ros2:nightly
+FROM ros:rolling-ros-base
 
 LABEL Description="This image is used to make ROS2 rolling based projects build faster using build tools" Vendor="Shadow Robot" Version="1.0"
 
@@ -18,6 +18,7 @@ ENV toolset_branch ${toolset_branch:-"master"}
 ENV server_type="docker_hub"
 ENV used_modules="check_cache,create_workspace"
 ENV ros_release_name=rolling
+ENV ubuntu_version_name=jammy
 ENV remote_shell_script="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/bin/sr-run-ci-build.sh"
 
 ENV remote_additional_bashrc_cmds="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/docker/utils/additional_bashrc_cmds_rolling"

docker/ros2/rolling/buildspec.yml

@@ -5,36 +5,27 @@ env:
   variables:
     branch: default
     image: build-tools
-    tag: focal-rolling
+    tag: rolling
     dockerfile_folder: docker/ros2/rolling/
   exported-variables:
     - branch
     - image
     - tag
     - dockerfile_folder
 phases:
-  install:
-    runtime-versions:
-      docker: 19
   build:
     commands:
       - |
         if [[ $branch != "default" ]]; then
           toolset_branch=$branch
         elif [[ $CODEBUILD_SOURCE_VERSION == "pr/"* ]]; then
-          pr_number=${CODEBUILD_SOURCE_VERSION#pr/}
           pr_number=${CODEBUILD_SOURCE_VERSION#pr/}
           branch=$(curl -s "https://api.github.com/repos/shadow-robot/sr-build-tools/pulls/$pr_number" | jq -r '.head.ref')
         elif [[ $CODEBUILD_SOURCE_VERSION =~ ^[0-9a-f]{40}$ ]]; then
           branch=$(git ls-remote --symref origin | grep -m1 $CODEBUILD_SOURCE_VERSION | awk -F '/' '{print $NF}')
         else
           branch=$CODEBUILD_SOURCE_VERSION
         fi
-      - awscli_v1_executable=$(which aws)
-      - rm $awscli_v1_executable
-      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-      - unzip awscliv2.zip
-      - ./aws/install
       - git checkout $branch
       - cd $dockerfile_folder
       - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot

docker/utils/additional_bashrc_cmds_humble

@@ -0,0 +1,49 @@
+# Print nickname for git/hg/bzr/svn version control in CWD
+# Optional $1 of format string for printf, default "(%s) "
+if [ -f /etc/bash_completion.d/git ]; then
+    . /etc/bash_completion.d/git
+fi
+if [ -f /etc/bash_completion.d/git-prompt ]; then
+    . /etc/bash_completion.d/git-prompt
+fi
+if [ -f /usr/share/bash-completion/completions/git ]; then
+    . /usr/share/bash-completion/completions/git
+fi
+function be_get_branch {
+  local dir="$PWD"
+  local vcs
+  local nick
+  while [[ "$dir" != "/" ]]; do
+    for vcs in git hg svn bzr; do
+      if [[ -d "$dir/.$vcs" ]] && hash "$vcs" &>/dev/null; then
+        case "$vcs" in
+          git) __git_ps1 "${1:-(%s) }"; return;;
+          hg) nick=$(hg branch 2>/dev/null);;
+          svn) nick=$(svn info 2>/dev/null\
+                | grep -e '^Repository Root:'\
+                | sed -e 's#.*/##');;
+          bzr)
+            local conf="${dir}/.bzr/branch/branch.conf" # normal branch
+            [[ -f "$conf" ]] && nick=$(grep -E '^nickname =' "$conf" | cut -d' ' -f 3)
+            conf="${dir}/.bzr/branch/location" # colo/lightweight branch
+            [[ -z "$nick" ]] && [[ -f "$conf" ]] && nick="$(basename "$(< $conf)")"
+            [[ -z "$nick" ]] && nick="$(basename "$(readlink -f "$dir")")";;
+        esac
+        [[ -n "$nick" ]] && printf "${1:-(%s) }" "$nick"
+        return 0
+      fi
+    done
+    dir="$(dirname "$dir")"
+  done
+}
+# Add branch to PS1 (based on $PS1 or $1), formatted as $2
+export GIT_PS1_SHOWDIRTYSTATE=yes
+
+#pretty PS1:
+export PS1="\$([[ \$? != 0 ]] && echo \"[\[\033[0;31m\]\342\234\227\[\033[0;37m\]]\342\224\200\")[$(if [[ ${EUID} == 0 ]]; then echo '\[\033[0;31m\]\h'; else echo '\[\033[0;33m\]\u\[\033[0;37m\]@\[\033[0;95m\]\h'; fi)\[\033[0;37m\]]\342\224\200[\[\033[0;32m\]\w\[\033[0;37m\]]\n\[\033[0;37m\]\342\224\224\342\224\200\342\224\200\342\225\274 \[\033[0m\]"
+export PS1="\[\033[0;37m\]\342\224\214\342\224\200\$(be_get_branch "$2")${PS1}";
+
+alias dephumble='rosdep install --from-paths . --ignore-src --rosdistro humble -y'
+
+export ROSCONSOLE_FORMAT='[${severity}](${node}): [${time}] ${message}'
+alias python=python3