From 24adc7b086661684d5526b531dada729b0e5ab5d Mon Sep 17 00:00:00 2001 From: Mateusz Szczygielski Date: Mon, 3 Jun 2024 15:29:05 +0200 Subject: [PATCH] Adjust CI --- .github/workflows/build-and-test.yml | 348 ++++++++++++++------------- 1 file changed, 175 insertions(+), 173 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 6d2dcc43..9f265e2e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -7,6 +7,7 @@ on: branches: - 'main' - 'develop' + - 'feature/rework-dockerfile' pull_request: types: [opened, synchronize, reopened] @@ -24,12 +25,12 @@ jobs: uses: actions/checkout@v3 with: clean: false - - name: Clean repository excluding taped test data - run: git clean -ffdx -e external/taped_test_data + - name: Clean repository excluding RGL blobs repo + run: git clean -ffdx -e external/rgl_blobs - name: Import private extensions run: vcs import < extensions.repos - - name: Install taped test dependencies - run: ./setup.py --install-taped-test-deps + - name: Install taped test runtime dependencies + run: ./setup.py --fetch-rgl-blobs load-env: uses: ./.github/workflows/load-env-subworkflow.yml @@ -37,182 +38,183 @@ jobs: ###### BUILD USING RGL DOCKER ###### build-core: needs: [checkout-repository, load-env] - uses: ./.github/workflows/build-subworkflow.yml - with: - build-command: ./setup.py --build-dir build-core - self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} - optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} - docker-image: localhost:5000/rgl:latest - - build-pcl: - needs: [checkout-repository, load-env] - uses: ./.github/workflows/build-subworkflow.yml - with: - build-command: './setup.py --with-pcl --build-taped-test --build-dir build-pcl' - self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} - optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} - docker-image: localhost:5000/rgl:latest - - build-ros2: - needs: [ checkout-repository, load-env ] - uses: ./.github/workflows/build-subworkflow.yml - with: - build-command: ' - source /opt/ros/humble/setup.bash && - ./setup.py --with-ros2-standalone --build-dir build-ros2' - self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} - optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} - docker-image: localhost:5000/rgl:latest - - build-udp: - needs: [ checkout-repository, load-env ] - uses: ./.github/workflows/build-subworkflow.yml - with: - build-command: './setup.py --with-udp --build-dir build-udp' - self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} - optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} - docker-image: localhost:5000/rgl:latest - - build-snow: - needs: [ checkout-repository, load-env ] - uses: ./.github/workflows/build-subworkflow.yml - with: - build-command: './setup.py --with-snow --build-dir build-snow' - self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} - optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} - docker-image: localhost:5000/rgl:latest - - build-all: - needs: [ checkout-repository, load-env ] - uses: ./.github/workflows/build-subworkflow.yml - with: - build-command: ' - source /opt/ros/humble/setup.bash && - ./setup.py --with-pcl --with-ros2-standalone --with-udp --with-snow --build-taped-test --build-dir build-all' - self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} - optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} - docker-image: localhost:5000/rgl:latest - -###### TEST WITH RGL DOCKER IMAGE ###### + runs-on: self-hosted + steps: + - run: ' + docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }} + --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1 + --build-arg BUILD_CMD="./setup.py" + --target=exporter --output=bin-core .' + +# build-pcl: +# needs: [checkout-repository, load-env] +# uses: ./.github/workflows/build-subworkflow.yml +# with: +# build-command: './setup.py --with-pcl --build-taped-test --build-dir build-pcl' +# self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} +# optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} +# docker-image: localhost:5000/rgl:latest +# +# build-ros2: +# needs: [ checkout-repository, load-env ] +# uses: ./.github/workflows/build-subworkflow.yml +# with: +# build-command: ' +# source /opt/ros/humble/setup.bash && +# ./setup.py --with-ros2-standalone --build-dir build-ros2' +# self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} +# optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} +# docker-image: localhost:5000/rgl:latest +# +# build-udp: +# needs: [ checkout-repository, load-env ] +# uses: ./.github/workflows/build-subworkflow.yml +# with: +# build-command: './setup.py --with-udp --build-dir build-udp' +# self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} +# optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} +# docker-image: localhost:5000/rgl:latest +# +# build-snow: +# needs: [ checkout-repository, load-env ] +# uses: ./.github/workflows/build-subworkflow.yml +# with: +# build-command: './setup.py --with-snow --build-dir build-snow' +# self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} +# optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} +# docker-image: localhost:5000/rgl:latest +# +# build-all: +# needs: [ checkout-repository, load-env ] +# uses: ./.github/workflows/build-subworkflow.yml +# with: +# build-command: ' +# source /opt/ros/humble/setup.bash && +# ./setup.py --with-pcl --with-ros2-standalone --with-udp --with-snow --build-taped-test --build-dir build-all' +# self-hosted-user-id: ${{ needs.load-env.outputs.user-id }} +# optix-install-dir: ${{ needs.load-env.outputs.optix-install-dir }} +# docker-image: localhost:5000/rgl:latest +# +####### TEST WITH RGL DOCKER IMAGE ###### test-core-dev: needs: [build-core] uses: ./.github/workflows/test-subworkflow.yml with: - test-command: 'cd build-core/test && ./RobotecGPULidar_test' - docker-image: localhost:5000/rgl:latest - - test-pcl-dev: - needs: [ build-pcl ] - uses: ./.github/workflows/test-subworkflow.yml - with: - test-command: ' - export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data && - cd build-pcl/test && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test' - docker-image: localhost:5000/rgl:latest - - test-ros2-dev: - needs: [ build-ros2 ] - uses: ./.github/workflows/test-subworkflow.yml - with: - # Source ROS2 and radar_msgs, standalone build is tested in prod environment - # Run tests twice, each for different RMW implementation - test-command: ' - source /opt/ros/humble/setup.bash && - source /rgldep/radar_msgs/install/setup.bash && - cd build-ros2/test && - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test' - docker-image: localhost:5000/rgl:latest - - test-udp-dev: - needs: [ build-udp ] - uses: ./.github/workflows/test-subworkflow.yml - with: - test-command: 'cd build-udp/test && ./RobotecGPULidar_test' - docker-image: localhost:5000/rgl:latest - - test-snow-dev: - needs: [ build-snow ] - uses: ./.github/workflows/test-subworkflow.yml - with: - test-command: 'cd build-snow/test && ./RobotecGPULidar_test' - docker-image: localhost:5000/rgl:latest - - test-all-dev: - needs: [ build-all ] - uses: ./.github/workflows/test-subworkflow.yml - with: - # Source ROS2 and radar_msgs, standalone build is tested in prod environment - # Set `RGL_TEST_VLP16_CALIB_FILE` for UDP-ROS2 integration test - # Run tests twice, each for different RMW implementation - test-command: ' - source /opt/ros/humble/setup.bash && - source /rgldep/radar_msgs/install/setup.bash && - export RGL_TEST_VLP16_CALIB_FILE=$(pwd)/extensions/udp/test/resources/Ros2Vlp16Calib.yaml && - export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data && - cd build-all/test && - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test && - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test' + test-command: 'cd bin-core && cp test/RobotecGPULidar_test . && ./RobotecGPULidar_test' docker-image: localhost:5000/rgl:latest - -###### TEST WITH CLEAN UBUNTU DOCKER IMAGE ###### +# +# test-pcl-dev: +# needs: [ build-pcl ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# test-command: ' +# export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data && +# cd build-pcl/test && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test' +# docker-image: localhost:5000/rgl:latest +# +# test-ros2-dev: +# needs: [ build-ros2 ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# # Source ROS2 and radar_msgs, standalone build is tested in prod environment +# # Run tests twice, each for different RMW implementation +# test-command: ' +# source /opt/ros/humble/setup.bash && +# source /rgldep/radar_msgs/install/setup.bash && +# cd build-ros2/test && +# export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && +# export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test' +# docker-image: localhost:5000/rgl:latest +# +# test-udp-dev: +# needs: [ build-udp ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# test-command: 'cd build-udp/test && ./RobotecGPULidar_test' +# docker-image: localhost:5000/rgl:latest +# +# test-snow-dev: +# needs: [ build-snow ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# test-command: 'cd build-snow/test && ./RobotecGPULidar_test' +# docker-image: localhost:5000/rgl:latest +# +# test-all-dev: +# needs: [ build-all ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# # Source ROS2 and radar_msgs, standalone build is tested in prod environment +# # Set `RGL_TEST_VLP16_CALIB_FILE` for UDP-ROS2 integration test +# # Run tests twice, each for different RMW implementation +# test-command: ' +# source /opt/ros/humble/setup.bash && +# source /rgldep/radar_msgs/install/setup.bash && +# export RGL_TEST_VLP16_CALIB_FILE=$(pwd)/extensions/udp/test/resources/Ros2Vlp16Calib.yaml && +# export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data && +# cd build-all/test && +# export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test && +# export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test' +# docker-image: localhost:5000/rgl:latest +# +####### TEST WITH CLEAN UBUNTU DOCKER IMAGE ###### test-core-prod: needs: [build-core] uses: ./.github/workflows/test-subworkflow.yml with: - test-command: 'cd build-core/test && ./RobotecGPULidar_test' - docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 - - test-pcl-prod: - needs: [ build-pcl ] - uses: ./.github/workflows/test-subworkflow.yml - with: - # Additionally, install PCL extension dependent libraries for runtime - test-command: ' - apt update && apt install -y libxcursor1 libgl1 && - export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data && - cd build-pcl/test && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test' - docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 - - test-ros2-prod: - needs: [ build-ros2 ] - uses: ./.github/workflows/test-subworkflow.yml - with: - # Copy ROS2 libraries to be visible for libRobotecGPULidar.so - # Run tests twice, each for different RMW implementation - test-command: ' - cd build-ros2/test && - cp -r ../ros2_standalone/*.so* ../ && - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test' - docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 - - test-udp-prod: - needs: [ build-udp ] - uses: ./.github/workflows/test-subworkflow.yml - with: - test-command: 'cd build-udp/test && ./RobotecGPULidar_test' - docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 - - test-snow-prod: - needs: [ build-snow ] - uses: ./.github/workflows/test-subworkflow.yml - with: - test-command: 'cd build-snow/test && ./RobotecGPULidar_test' - docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 - - test-all-prod: - needs: [ build-all ] - uses: ./.github/workflows/test-subworkflow.yml - with: - # Install PCL extension dependent libraries for runtime - # Copy ROS2 libraries to be visible for libRobotecGPULidar.so - # Run tests twice, each for different RMW implementation - test-command: ' - apt update && apt install -y libxcursor1 libgl1 && - export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data && - cd build-all/test && - cp -r ../ros2_standalone/*.so* ../ && - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test && - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test' + test-command: 'cd bin-core && cp test/RobotecGPULidar_test . && ./RobotecGPULidar_test' docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 +# +# test-pcl-prod: +# needs: [ build-pcl ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# # Additionally, install PCL extension dependent libraries for runtime +# test-command: ' +# apt update && apt install -y libxcursor1 libgl1 && +# export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data && +# cd build-pcl/test && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test' +# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 +# +# test-ros2-prod: +# needs: [ build-ros2 ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# # Copy ROS2 libraries to be visible for libRobotecGPULidar.so +# # Run tests twice, each for different RMW implementation +# test-command: ' +# cd build-ros2/test && +# cp -r ../ros2_standalone/*.so* ../ && +# export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && +# export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test' +# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 +# +# test-udp-prod: +# needs: [ build-udp ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# test-command: 'cd build-udp/test && ./RobotecGPULidar_test' +# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 +# +# test-snow-prod: +# needs: [ build-snow ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# test-command: 'cd build-snow/test && ./RobotecGPULidar_test' +# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 +# +# test-all-prod: +# needs: [ build-all ] +# uses: ./.github/workflows/test-subworkflow.yml +# with: +# # Install PCL extension dependent libraries for runtime +# # Copy ROS2 libraries to be visible for libRobotecGPULidar.so +# # Run tests twice, each for different RMW implementation +# test-command: ' +# apt update && apt install -y libxcursor1 libgl1 && +# export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/taped_test_data && +# cd build-all/test && +# cp -r ../ros2_standalone/*.so* ../ && +# export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test && +# export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./taped_test/RobotecGPULidar_taped_test' +# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04