From 13bbc2772f0c71412a229b4bcde77ed351939a42 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 2 Jul 2024 15:35:22 +0200 Subject: [PATCH 1/3] Implement the test_require_gpu across ci jobs Signed-off-by: Jose Luis Rivero --- doc/configuration_options.rst | 2 ++ ros_buildfarm/ci_job.py | 2 ++ ros_buildfarm/config/ci_build_file.py | 5 +++++ ros_buildfarm/scripts/devel/build_and_test.py | 2 -- ros_buildfarm/templates/ci/ci_job.xml.em | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/configuration_options.rst b/doc/configuration_options.rst index 12f356cb8..e7e7c4685 100644 --- a/doc/configuration_options.rst +++ b/doc/configuration_options.rst @@ -598,6 +598,8 @@ The following options are valid in version ``1`` (beside the generic options): By default, the resulting archives are only available to other jobs within Jenkins. +* ``tests_require_gpu``: a boolean flag to indicate if software tests needs gpu + support to run correctly. The following options are valid as keys in the ``_config`` dict under ``targets``: diff --git a/ros_buildfarm/ci_job.py b/ros_buildfarm/ci_job.py index d6d8dd8ac..311b43750 100644 --- a/ros_buildfarm/ci_job.py +++ b/ros_buildfarm/ci_job.py @@ -288,6 +288,8 @@ def _get_ci_job_config( 'ros_version': ros_version, 'build_environment_variables': build_environment_variables, + 'require_gpu_support': build_file.tests_require_gpu_default, + 'timeout_minutes': build_file.jenkins_job_timeout, 'repos_file_urls': repos_files, diff --git a/ros_buildfarm/config/ci_build_file.py b/ros_buildfarm/config/ci_build_file.py index 12b32a078..2fbcff630 100644 --- a/ros_buildfarm/config/ci_build_file.py +++ b/ros_buildfarm/config/ci_build_file.py @@ -136,6 +136,11 @@ def __init__(self, name, data): # noqa: D107 self.show_plots[plot_group].append( PlotConfig(name, plot_config_data)) + self.tests_require_gpu_default = False + if 'tests_require_gpu' in data: + self.tests_require_gpu_default = bool( + data['tests_require_gpu']) + self.benchmark_patterns = [] if 'benchmark_patterns' in data: self.benchmark_patterns = data['benchmark_patterns'] diff --git a/ros_buildfarm/scripts/devel/build_and_test.py b/ros_buildfarm/scripts/devel/build_and_test.py index 924c5e798..4ce486008 100644 --- a/ros_buildfarm/scripts/devel/build_and_test.py +++ b/ros_buildfarm/scripts/devel/build_and_test.py @@ -19,7 +19,6 @@ from ros_buildfarm.argument import add_argument_build_tool from ros_buildfarm.argument import add_argument_build_tool_args from ros_buildfarm.argument import add_argument_build_tool_test_args -from ros_buildfarm.argument import add_argument_require_gpu_support from ros_buildfarm.argument import add_argument_ros_version from ros_buildfarm.argument import extract_multiple_remainders from ros_buildfarm.common import Scope @@ -58,7 +57,6 @@ def main(argv=sys.argv[1:]): action='store_true', help='The flag if the workspace should be cleaned after the ' 'invocation') - add_argument_require_gpu_support(parser) remainder_args = extract_multiple_remainders(argv, (a1, a2)) args = parser.parse_args(argv) diff --git a/ros_buildfarm/templates/ci/ci_job.xml.em b/ros_buildfarm/templates/ci/ci_job.xml.em index 6d1ed4c99..565abb9a4 100644 --- a/ros_buildfarm/templates/ci/ci_job.xml.em +++ b/ros_buildfarm/templates/ci/ci_job.xml.em @@ -421,11 +421,13 @@ parameters = [ 'export UNDERLAY%d_JOB_SPACE=$WORKSPACE/underlay%d/ros%d-linux' % (i + 1, i + 1, local_ros_version) for i, local_ros_version in zip(range(len(underlay_source_jobs)), [ros_version] * len(underlay_source_jobs)) ] + [ + ('if [ ! -c /dev/nvidia[0-9] ]; then echo "--require-gpu-support is enabled but can not detect nvidia support installed" && exit 1; fi' if require_gpu_support else ''), 'rm -fr $WORKSPACE/ws/test_results', 'mkdir -p $WORKSPACE/ws/test_results', '# If using Podman, change the user namespace to preserve UID. No effect if using Docker.', 'export PODMAN_USERNS=keep-id', 'docker run' + + (' --env=DISPLAY=:0.0 --env=QT_X11_NO_MITSHM=1 --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw --gpus all' if require_gpu_support else '') + ' --rm ' + ' --cidfile=$WORKSPACE/docker_build_and_test/docker.cid' + ((' -e CCACHE_DIR=/home/buildfarm/.ccache' + From b1e42cf908a399dd33eadc3e595714d48718aaf0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 26 Jun 2024 19:02:46 +0200 Subject: [PATCH 2/3] Existing broken code that required to be removed Signed-off-by: Jose Luis Rivero --- ros_buildfarm/scripts/ci/build_task_generator.py | 3 --- ros_buildfarm/templates/devel/devel_task.Dockerfile.em | 2 -- 2 files changed, 5 deletions(-) diff --git a/ros_buildfarm/scripts/ci/build_task_generator.py b/ros_buildfarm/scripts/ci/build_task_generator.py index 88bb423f3..ee0e3f1ae 100644 --- a/ros_buildfarm/scripts/ci/build_task_generator.py +++ b/ros_buildfarm/scripts/ci/build_task_generator.py @@ -29,7 +29,6 @@ from ros_buildfarm.argument import add_argument_install_packages from ros_buildfarm.argument import add_argument_os_code_name from ros_buildfarm.argument import add_argument_os_name -from ros_buildfarm.argument import add_argument_require_gpu_support from ros_buildfarm.argument import add_argument_ros_version from ros_buildfarm.argument import add_argument_rosdistro_name from ros_buildfarm.argument import add_argument_run_abichecker @@ -61,7 +60,6 @@ def main(argv=sys.argv[1:]): add_argument_install_packages(parser) add_argument_ros_version(parser) add_argument_run_abichecker(parser) - add_argument_require_gpu_support(parser) add_argument_testing(parser) parser.add_argument( '--workspace-root', nargs='+', @@ -132,7 +130,6 @@ def main(argv=sys.argv[1:]): 'testing': args.testing, 'run_abichecker': args.run_abichecker, - 'require_gpu_support': args.require_gpu_support, 'workspace_root': mapped_workspaces[-1][1], 'parent_result_space': [mapping[1] for mapping in mapped_workspaces[:-1]], } diff --git a/ros_buildfarm/templates/devel/devel_task.Dockerfile.em b/ros_buildfarm/templates/devel/devel_task.Dockerfile.em index 5f9c02437..aac03ff29 100644 --- a/ros_buildfarm/templates/devel/devel_task.Dockerfile.em +++ b/ros_buildfarm/templates/devel/devel_task.Dockerfile.em @@ -129,8 +129,6 @@ else: ' /tmp/ros_buildfarm/scripts/devel/build_and_test.py' + \ ' --rosdistro-name %s' % (rosdistro_name or "''") + \ ' --ros-version ' + str(ros_version) - if require_gpu_support: - cmd += ' --require-gpu-support' cmd += \ ' --build-tool ' + build_tool + \ ' --workspace-root ' + workspace_root + \ From b3670138bb7d9f5cd1647aafc6898219a57af572 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 29 Jul 2024 20:29:14 +0200 Subject: [PATCH 3/3] Do not break API but use deprecation warnings Signed-off-by: Jose Luis Rivero --- ros_buildfarm/scripts/ci/build_task_generator.py | 8 ++++++++ ros_buildfarm/scripts/devel/build_and_test.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/ros_buildfarm/scripts/ci/build_task_generator.py b/ros_buildfarm/scripts/ci/build_task_generator.py index ee0e3f1ae..a2ab89804 100644 --- a/ros_buildfarm/scripts/ci/build_task_generator.py +++ b/ros_buildfarm/scripts/ci/build_task_generator.py @@ -29,6 +29,7 @@ from ros_buildfarm.argument import add_argument_install_packages from ros_buildfarm.argument import add_argument_os_code_name from ros_buildfarm.argument import add_argument_os_name +from ros_buildfarm.argument import add_argument_require_gpu_support from ros_buildfarm.argument import add_argument_ros_version from ros_buildfarm.argument import add_argument_rosdistro_name from ros_buildfarm.argument import add_argument_run_abichecker @@ -60,6 +61,7 @@ def main(argv=sys.argv[1:]): add_argument_install_packages(parser) add_argument_ros_version(parser) add_argument_run_abichecker(parser) + add_argument_require_gpu_support(parser) add_argument_testing(parser) parser.add_argument( '--workspace-root', nargs='+', @@ -70,6 +72,12 @@ def main(argv=sys.argv[1:]): for k, v in remainder_args.items(): setattr(args, k, v) + if args.require_gpu_support: + print( + 'WARNING: using the --require-gpu-support argument is deprecated.' + 'Can be removed without changing functionality.', + file=sys.stderr) + apt_cache = Cache() debian_pkg_names = set(['build-essential']) diff --git a/ros_buildfarm/scripts/devel/build_and_test.py b/ros_buildfarm/scripts/devel/build_and_test.py index 4ce486008..6c4234b98 100644 --- a/ros_buildfarm/scripts/devel/build_and_test.py +++ b/ros_buildfarm/scripts/devel/build_and_test.py @@ -19,6 +19,7 @@ from ros_buildfarm.argument import add_argument_build_tool from ros_buildfarm.argument import add_argument_build_tool_args from ros_buildfarm.argument import add_argument_build_tool_test_args +from ros_buildfarm.argument import add_argument_require_gpu_support from ros_buildfarm.argument import add_argument_ros_version from ros_buildfarm.argument import extract_multiple_remainders from ros_buildfarm.common import Scope @@ -57,12 +58,19 @@ def main(argv=sys.argv[1:]): action='store_true', help='The flag if the workspace should be cleaned after the ' 'invocation') + add_argument_require_gpu_support(parser) remainder_args = extract_multiple_remainders(argv, (a1, a2)) args = parser.parse_args(argv) for k, v in remainder_args.items(): setattr(args, k, v) + if args.require_gpu_support: + print( + 'WARNING: using the --require-gpu-support argument is deprecated.' + 'Can be removed without changing functionality.', + file=sys.stderr) + ensure_workspace_exists(args.workspace_root) if args.clean_before: