From 660767730a24083d437b53103218bc6c8cd96871 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Sun, 24 Mar 2024 21:27:37 -0400 Subject: [PATCH 01/26] python3.12 support --- .github/workflows/macos.yml | 8 +++++-- .github/workflows/ubuntu-wheel.yml | 12 ++++++++-- .github/workflows/windows.yml | 8 +++++-- CHANGELOG.md | 1 + docker/Dockerfile.wheel | 2 +- docker/README.md | 2 +- docker/docker_build.sh | 36 +++++++++++++++++++++++++++++- docker/docker_test.sh | 24 ++++++++++++++++++++ docs/arm.rst | 1 + docs/getting_started.in.rst | 5 +++++ python/README.rst | 1 + python/setup.py | 1 + util/ci_utils.sh | 6 ++--- 13 files changed, 95 insertions(+), 12 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 00bd9636130..77771a95a7a 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -127,7 +127,7 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] is_main: - ${{ github.ref == 'refs/heads/main' }} exclude: @@ -137,6 +137,8 @@ jobs: python_version: '3.9' - is_main: false python_version: '3.10' + - is_main: false + python_version: '3.11' env: BUILD_CUDA_MODULE: OFF @@ -234,7 +236,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] is_main: - ${{ github.ref == 'refs/heads/main' }} exclude: @@ -244,6 +246,8 @@ jobs: python_version: '3.9' - is_main: false python_version: '3.10' + - is_main: false + python_version: '3.11' env: OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index a798762c965..e7da0ea3fa5 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] is_main: - ${{ github.ref == 'refs/heads/main' }} exclude: @@ -40,6 +40,8 @@ jobs: python_version: '3.9' - is_main: false python_version: '3.10' + - is_main: false + python_version: '3.11' env: DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} PYTHON_VERSION: ${{ matrix.python_version }} @@ -64,6 +66,8 @@ jobs: docker/docker_build.sh cuda_wheel_py310_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py311_dev + elif [ "${{ env.PYTHON_VERSION }}" = "3.12" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then + docker/docker_build.sh cuda_wheel_py312_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.8" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_build.sh cuda_wheel_py38 elif [ "${{ env.PYTHON_VERSION }}" = "3.9" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then @@ -72,6 +76,8 @@ jobs: docker/docker_build.sh cuda_wheel_py310 elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_build.sh cuda_wheel_py311 + elif [ "${{ env.PYTHON_VERSION }}" = "3.12" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then + docker/docker_build.sh cuda_wheel_py312 fi PIP_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d-[0-9]*.whl)" PIP_CPU_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d_cpu*.whl)" @@ -117,7 +123,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] is_main: - ${{ github.ref == 'refs/heads/main' }} exclude: @@ -127,6 +133,8 @@ jobs: python_version: '3.9' - is_main: false python_version: '3.10' + - is_main: false + python_version: '3.11' env: OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML steps: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d687f6b7e33..9ca74f0f657 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -242,7 +242,7 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] is_main: - ${{ github.ref == 'refs/heads/main' }} exclude: @@ -252,6 +252,8 @@ jobs: python_version: '3.9' - is_main: false python_version: '3.10' + - is_main: false + python_version: '3.11' steps: - name: Checkout source code @@ -347,7 +349,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] is_main: - ${{ github.ref == 'refs/heads/main' }} exclude: @@ -357,6 +359,8 @@ jobs: python_version: '3.9' - is_main: false python_version: '3.10' + - is_main: false + python_version: '3.11' steps: - name: Checkout source code diff --git a/CHANGELOG.md b/CHANGELOG.md index 3509dda95e6..b8d71ddd4f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ - Fix regression in printing cuda tensor from PR #6444 🐛 - Add Python pathlib support for file IO (PR #6619) - Fix log error message for `probability` argument validation in `PointCloud::SegmentPlane` (PR #6622) +- Python 3.12 support ## 0.13 diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index 6a7d7080aed..98ec8d52be5 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -1,5 +1,5 @@ # FROM must be called before other ARGS except for ARG BASE_IMAGE -ARG BASE_IMAGE=nvidia/cuda:11.7.1-cudnn8-devel-ubuntu18.04 +ARG BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu18.04 FROM ${BASE_IMAGE} # Customizable build arguments from cuda.yml diff --git a/docker/README.md b/docker/README.md index d14d9521c19..732eebf4bda 100644 --- a/docker/README.md +++ b/docker/README.md @@ -26,7 +26,7 @@ to install Nvidia Docker to run the CUDA container. To verify that the Nvidia Docker is working, run: ```bash -docker run --rm --gpus all nvidia/cuda:11.7-base nvidia-smi +docker run --rm --gpus all nvidia/cuda:11.8-base nvidia-smi ``` ### ARM64 Docker diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 33ab6ad0115..b78d56de73d 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -27,20 +27,24 @@ OPTION: openblas-amd64-py39-dev : OpenBLAS AMD64 3.9 wheel, developer mode openblas-amd64-py310-dev : OpenBLAS AMD64 3.10 wheel, developer mode openblas-amd64-py311-dev : OpenBLAS AMD64 3.11 wheel, developer mode + openblas-amd64-py312-dev : OpenBLAS AMD64 3.12 wheel, developer mode openblas-amd64-py38 : OpenBLAS AMD64 3.8 wheel, release mode openblas-amd64-py39 : OpenBLAS AMD64 3.9 wheel, release mode openblas-amd64-py310 : OpenBLAS AMD64 3.10 wheel, release mode openblas-amd64-py311 : OpenBLAS AMD64 3.11 wheel, release mode + openblas-amd64-py312 : OpenBLAS AMD64 3.12 wheel, release mode # OpenBLAS ARM64 (Dockerfile.openblas) openblas-arm64-py38-dev : OpenBLAS ARM64 3.8 wheel, developer mode openblas-arm64-py39-dev : OpenBLAS ARM64 3.9 wheel, developer mode openblas-arm64-py310-dev : OpenBLAS ARM64 3.10 wheel, developer mode openblas-arm64-py311-dev : OpenBLAS ARM64 3.11 wheel, developer mode + openblas-arm64-py312-dev : OpenBLAS ARM64 3.12 wheel, developer mode openblas-arm64-py38 : OpenBLAS ARM64 3.8 wheel, release mode openblas-arm64-py39 : OpenBLAS ARM64 3.9 wheel, release mode openblas-arm64-py310 : OpenBLAS ARM64 3.10 wheel, release mode openblas-arm64-py311 : OpenBLAS ARM64 3.11 wheel, release mode + openblas-arm64-py312 : OpenBLAS ARM64 3.12 wheel, release mode # Ubuntu CPU CI (Dockerfile.ci) cpu-static : Ubuntu CPU static @@ -66,10 +70,12 @@ OPTION: cuda_wheel_py39_dev : CUDA Python 3.9 wheel, developer mode cuda_wheel_py310_dev : CUDA Python 3.10 wheel, developer mode cuda_wheel_py311_dev : CUDA Python 3.11 wheel, developer mode + cuda_wheel_py312_dev : CUDA Python 3.12 wheel, developer mode cuda_wheel_py38 : CUDA Python 3.8 wheel, release mode cuda_wheel_py39 : CUDA Python 3.9 wheel, release mode cuda_wheel_py310 : CUDA Python 3.10 wheel, release mode cuda_wheel_py311 : CUDA Python 3.11 wheel, release mode + cuda_wheel_py312 : CUDA Python 3.12 wheel, release mode " HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pwd)" @@ -78,7 +84,7 @@ HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pw CCACHE_VERSION=4.3 CMAKE_VERSION=cmake-3.20.6-linux-x86_64 CMAKE_VERSION_AARCH64=cmake-3.20.6-linux-aarch64 -CUDA_VERSION=11.7.1-cudnn8 +CUDA_VERSION=11.8.0-cudnn8 print_usage_and_exit_docker_build() { echo "$__usage_docker_build" @@ -127,6 +133,9 @@ openblas_export_env() { elif [[ "py311" =~ ^($options)$ ]]; then export PYTHON_VERSION=3.11 export DOCKER_TAG=${DOCKER_TAG}-py311 + elif [[ "py312" =~ ^($options)$ ]]; then + export PYTHON_VERSION=3.12 + export DOCKER_TAG=${DOCKER_TAG}-py312 else echo "Invalid python version." print_usage_and_exit_docker_build @@ -182,6 +191,8 @@ cuda_wheel_build() { PYTHON_VERSION=3.10 elif [[ "py311" =~ ^($options)$ ]]; then PYTHON_VERSION=3.11 + elif [[ "py312" =~ ^($options)$ ]]; then + PYTHON_VERSION=3.12 else echo "Invalid python version." print_usage_and_exit_docker_build @@ -490,6 +501,10 @@ function main() { openblas_export_env amd64 py311 dev openblas_build ;; + openblas-amd64-py312-dev) + openblas_export_env amd64 py312 dev + openblas_build + ;; openblas-amd64-py38) openblas_export_env amd64 py38 openblas_build @@ -506,6 +521,10 @@ function main() { openblas_export_env amd64 py311 openblas_build ;; + openblas-amd64-py312) + openblas_export_env amd64 py312 + openblas_build + ;; # OpenBLAS ARM64 openblas-arm64-py38-dev) @@ -524,6 +543,10 @@ function main() { openblas_export_env arm64 py311 dev openblas_build ;; + openblas-arm64-py312-dev) + openblas_export_env arm64 py312 dev + openblas_build + ;; openblas-arm64-py38) openblas_export_env arm64 py38 openblas_build @@ -540,6 +563,10 @@ function main() { openblas_export_env arm64 py311 openblas_build ;; + openblas-arm64-py312) + openblas_export_env arm64 py312 + openblas_build + ;; # CPU CI cpu-static) @@ -586,6 +613,9 @@ function main() { cuda_wheel_py311_dev) cuda_wheel_build py311 dev ;; + cuda_wheel_py312_dev) + cuda_wheel_build py312 dev + ;; cuda_wheel_py38) cuda_wheel_build py38 ;; @@ -598,6 +628,10 @@ function main() { cuda_wheel_py311) cuda_wheel_build py311 ;; + cuda_wheel_py312) + cuda_wheel_build py312 + ;; + # ML CIs 2-bionic) diff --git a/docker/docker_test.sh b/docker/docker_test.sh index 544eeb755e4..477496230d6 100755 --- a/docker/docker_test.sh +++ b/docker/docker_test.sh @@ -20,20 +20,24 @@ OPTION: openblas-amd64-py39-dev : OpenBLAS AMD64 3.9 wheel, developer mode openblas-amd64-py310-dev : OpenBLAS AMD64 3.10 wheel, developer mode openblas-amd64-py311-dev : OpenBLAS AMD64 3.11 wheel, developer mode + openblas-amd64-py312-dev : OpenBLAS AMD64 3.12 wheel, developer mode openblas-amd64-py38 : OpenBLAS AMD64 3.8 wheel, release mode openblas-amd64-py39 : OpenBLAS AMD64 3.9 wheel, release mode openblas-amd64-py310 : OpenBLAS AMD64 3.10 wheel, release mode openblas-amd64-py311 : OpenBLAS AMD64 3.11 wheel, release mode + openblas-amd64-py312 : OpenBLAS AMD64 3.12 wheel, release mode # OpenBLAS ARM64 (Dockerfile.openblas) openblas-arm64-py38-dev : OpenBLAS ARM64 3.8 wheel, developer mode openblas-arm64-py39-dev : OpenBLAS ARM64 3.9 wheel, developer mode openblas-arm64-py310-dev : OpenBLAS ARM64 3.10 wheel, developer mode openblas-arm64-py311-dev : OpenBLAS ARM64 3.11 wheel, developer mode + openblas-arm64-py312-dev : OpenBLAS ARM64 3.12 wheel, developer mode openblas-arm64-py38 : OpenBLAS ARM64 3.8 wheel, release mode openblas-arm64-py39 : OpenBLAS ARM64 3.9 wheel, release mode openblas-arm64-py310 : OpenBLAS ARM64 3.10 wheel, release mode openblas-arm64-py311 : OpenBLAS ARM64 3.11 wheel, release mode + openblas-arm64-py312 : OpenBLAS ARM64 3.12 wheel, release mode # Ubuntu CPU CI (Dockerfile.ci) cpu-static : Ubuntu CPU static @@ -230,6 +234,11 @@ openblas-amd64-py311-dev) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-amd64-py312-dev) + openblas_export_env amd64 py312 dev + openblas_print_env + cpp_python_linking_uninstall_test + ;; openblas-amd64-py38) openblas_export_env amd64 py38 openblas_print_env @@ -250,6 +259,11 @@ openblas-amd64-py311) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-amd64-py312) + openblas_export_env amd64 py312 + openblas_print_env + cpp_python_linking_uninstall_test + ;; # OpenBLAS ARM64 openblas-arm64-py38-dev) @@ -272,6 +286,11 @@ openblas-arm64-py311-dev) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-arm64-py312-dev) + openblas_export_env arm64 py312 dev + openblas_print_env + cpp_python_linking_uninstall_test + ;; openblas-arm64-py38) openblas_export_env arm64 py38 openblas_print_env @@ -292,6 +311,11 @@ openblas-arm64-py311) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-arm64-py312) + openblas_export_env arm64 py312 + openblas_print_env + cpp_python_linking_uninstall_test + ;; # CPU CI cpu-static) diff --git a/docs/arm.rst b/docs/arm.rst index e9f9c942413..93d5b47f126 100644 --- a/docs/arm.rst +++ b/docs/arm.rst @@ -71,6 +71,7 @@ commands: ./docker_build.sh openblas-arm64-py39 # Python 3.9 ./docker_build.sh openblas-arm64-py310 # Python 3.10 ./docker_build.sh openblas-arm64-py311 # Python 3.11 + ./docker_build.sh openblas-arm64-py311 # Python 3.12 After running ``docker_build.sh``, you shall see a ``.whl`` file generated the current directly on the host. Then simply install the ``.whl`` file by: diff --git a/docs/getting_started.in.rst b/docs/getting_started.in.rst index b9f390ea835..34f69dfdfe6 100644 --- a/docs/getting_started.in.rst +++ b/docs/getting_started.in.rst @@ -17,6 +17,7 @@ Supported Python versions: * 3.9 * 3.10 * 3.11 +* 3.12 Supported operating systems: @@ -74,24 +75,28 @@ version (``HEAD`` of ``main`` branch): - `Python 3.9 `__ - `Python 3.10 `__ - `Python 3.11 `__ + - `Python 3.12 `__ * - Linux (CPU) - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ - `Python 3.11 `__ + - `Python 3.12 `__ * - MacOS - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ - `Python 3.11 `__ + - `Python 3.12 `__ * - Windows - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ - `Python 3.11 `__ + - `Python 3.12 `__ Please use these links from the `latest version of this page `__ only. You can also diff --git a/python/README.rst b/python/README.rst index 232c3b0ca24..bf046cc2b11 100644 --- a/python/README.rst +++ b/python/README.rst @@ -48,6 +48,7 @@ With Python versions: * 3.9 * 3.10 * 3.11 +* 3.12 Resources ====================== diff --git a/python/setup.py b/python/setup.py index 029c5717180..59aa268dccc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -131,6 +131,7 @@ def finalize_options(self): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Education", "Topic :: Multimedia :: Graphics :: 3D Modeling", "Topic :: Multimedia :: Graphics :: 3D Rendering", diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 38ff654880e..3bcbfdbfd0c 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -25,10 +25,10 @@ LOW_MEM_USAGE=${LOW_MEM_USAGE:-OFF} # Dependency versions: # CUDA: see docker/docker_build.sh # ML -TENSORFLOW_VER="2.13.0" -TORCH_VER="2.0.1" +TENSORFLOW_VER="2.16.1" +TORCH_VER="2.2.0" TORCH_CPU_GLNX_VER="${TORCH_VER}+cpu" -TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu117" # match CUDA_VERSION in docker/docker_build.sh +TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu118" # match CUDA_VERSION in docker/docker_build.sh TORCH_MACOS_VER="${TORCH_VER}" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python From 4ba2d39faa53b4251862a2ef5bce1a07654b0aeb Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Sun, 24 Mar 2024 21:45:01 -0400 Subject: [PATCH 02/26] tensorboard bump --- python/requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 4a2f418b6bb..902f88f240d 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -1,6 +1,6 @@ pytest==7.1.2 pytest-randomly==3.8.0 scipy==1.10.1 -tensorboard==2.13.0 +tensorboard==2.16.1gs oauthlib==3.2.2 certifi==2023.7.22 From 17baa25de34ae9888fce1324b4839f949bb45833 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Sun, 24 Mar 2024 21:46:47 -0400 Subject: [PATCH 03/26] typo --- python/requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 902f88f240d..5c8186291af 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -1,6 +1,6 @@ pytest==7.1.2 pytest-randomly==3.8.0 scipy==1.10.1 -tensorboard==2.16.1gs +tensorboard==2.16.1s oauthlib==3.2.2 certifi==2023.7.22 From fae3dff86def8ebb005902679b5264623f28ce64 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Mon, 25 Mar 2024 08:50:42 -0400 Subject: [PATCH 04/26] what is happening with my typo fixing --- python/requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 5c8186291af..72d5784c5e8 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -1,6 +1,6 @@ pytest==7.1.2 pytest-randomly==3.8.0 scipy==1.10.1 -tensorboard==2.16.1s +tensorboard==2.16.1 oauthlib==3.2.2 certifi==2023.7.22 From 8a2cc17766132a676f4c8418b75c9596532a4d59 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Mon, 25 Mar 2024 09:07:27 -0400 Subject: [PATCH 05/26] bump scipy --- python/requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 72d5784c5e8..253ffee771c 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -1,6 +1,6 @@ pytest==7.1.2 pytest-randomly==3.8.0 -scipy==1.10.1 +scipy==1.11.4 tensorboard==2.16.1 oauthlib==3.2.2 certifi==2023.7.22 From 66f8800a8640790c02ea0842b4f7ca33a5894864 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Mon, 25 Mar 2024 09:18:53 -0400 Subject: [PATCH 06/26] unrelated style check fix --- examples/python/reconstruction_system/make_fragments.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/python/reconstruction_system/make_fragments.py b/examples/python/reconstruction_system/make_fragments.py index 17d875aa86c..5af9d960f4b 100644 --- a/examples/python/reconstruction_system/make_fragments.py +++ b/examples/python/reconstruction_system/make_fragments.py @@ -145,6 +145,7 @@ def make_pointcloud_for_fragment(path_dataset, color_files, depth_files, write_ascii=False, compressed=True) + def process_single_fragment(fragment_id, color_files, depth_files, n_files, n_fragments, config): if config["path_intrinsic"]: From 53036135c6390a2bae69a666aa2018876620d362 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Mon, 25 Mar 2024 11:48:07 -0400 Subject: [PATCH 07/26] scipy 1.11.4 required for python3.12, otherwise unchanged --- python/requirements_test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 253ffee771c..c41e7ce5753 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -1,6 +1,7 @@ pytest==7.1.2 pytest-randomly==3.8.0 -scipy==1.11.4 +scipy==1.10.1; python_version < "3.12" +scipy==1.11.4; python_version >= "3.12" tensorboard==2.16.1 oauthlib==3.2.2 certifi==2023.7.22 From e5dd29c1abf1bea734e7efd295f0e8b70fdc5ecb Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Mon, 25 Mar 2024 11:52:57 -0400 Subject: [PATCH 08/26] tensorboard 2.16.2 required for python3.12, otherwise unchanged --- python/requirements_test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/requirements_test.txt b/python/requirements_test.txt index c41e7ce5753..7af2d222c27 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -2,6 +2,7 @@ pytest==7.1.2 pytest-randomly==3.8.0 scipy==1.10.1; python_version < "3.12" scipy==1.11.4; python_version >= "3.12" -tensorboard==2.16.1 +tensorboard==2.13.0; python_version < "3.12" +tensorboard==2.16.2; python_version >= "3.12" oauthlib==3.2.2 certifi==2023.7.22 From 15abf1898bb711c5ae43e2f7f27bd87f26aae5a0 Mon Sep 17 00:00:00 2001 From: Stuart Wheaton Date: Mon, 25 Mar 2024 20:44:38 -0400 Subject: [PATCH 09/26] fix improper error fstring --- python/open3d/visualization/tensorboard_plugin/summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/open3d/visualization/tensorboard_plugin/summary.py b/python/open3d/visualization/tensorboard_plugin/summary.py index 0b6e254b754..4bb35fa5da7 100644 --- a/python/open3d/visualization/tensorboard_plugin/summary.py +++ b/python/open3d/visualization/tensorboard_plugin/summary.py @@ -424,7 +424,7 @@ def get_or_check_shape(prop, tensor_tuple, exp_shape): raise ValueError( f"Property {prop} tensor should have shape[{k}]" f"={s} for all elements but is " - f"{tensor.shape[k-1] for tensor in tensor_tuple}.") + f"{[tensor.shape[k-1] for tensor in tensor_tuple]}.") return shape[:2] From d96ecbfb66d9b682351bda36f62401b5b5eb3267 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 16 Jul 2024 10:37:13 -0700 Subject: [PATCH 10/26] Update windows CUDA version to match Linux, TF version, torch as latest supporting Intel Mac, update werkzeug, flask to matching versions. --- .github/workflows/windows.yml | 2 +- python/requirements.txt | 3 ++- util/ci_utils.sh | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 3b738c288e8..65e1b216e1f 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -47,7 +47,7 @@ jobs: STATIC_RUNTIME: ON include: - BUILD_CUDA_MODULE: ON - CUDA_VERSION: 11.0.3 + CUDA_VERSION: 11.8.0 env: BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} diff --git a/python/requirements.txt b/python/requirements.txt index 148650326df..a7da4963446 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,5 +1,6 @@ numpy>=1.18.0,<2.0.0 dash>=2.6.0 -werkzeug>=2.2.3 +werkzeug>=3.0.0 +flask>=3.0.0 nbformat>=5.7.0 configargparse diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 6e77244a429..565cc09161c 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -25,8 +25,8 @@ LOW_MEM_USAGE=${LOW_MEM_USAGE:-OFF} # Dependency versions: # CUDA: see docker/docker_build.sh # ML -TENSORFLOW_VER="2.16.1" -TORCH_VER="2.2.0" +TENSORFLOW_VER="2.16.2" +TORCH_VER="2.2.2" TORCH_CPU_GLNX_VER="${TORCH_VER}+cpu" TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu118" # match CUDA_VERSION in docker/docker_build.sh TORCH_MACOS_VER="${TORCH_VER}" From 9e1b5ae68f94c7f26dfc5f796f2eb55bde00c193 Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Wed, 14 Aug 2024 06:12:52 -0700 Subject: [PATCH 11/26] use c++17 if pytorch ops are enabled --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e4781e7e72..57751184320 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -291,8 +291,8 @@ endif() # Global flag to set CXX standard. # This does not affect 3rd party libraries. # Tensorflow 2.9+ requires cxx_17, but MSVC 19.29 throws errors with C++17 -# enabled. -if (BUILD_SYCL_MODULE OR BUILD_TENSORFLOW_OPS) +# enabled. ATen in Pytorch 2.2 requires cxx_17 +if (BUILD_SYCL_MODULE OR BUILD_TENSORFLOW_OPS OR BUILD_PYTORCH_OPS) set(CMAKE_CXX_STANDARD 17) else() set(CMAKE_CXX_STANDARD 14) From 1ff496f602733a3cbdf5f44d1ae9e1737749164c Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Wed, 14 Aug 2024 06:40:05 -0700 Subject: [PATCH 12/26] update url for Windows CUDA installer --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 65e1b216e1f..807bc5be5be 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -70,7 +70,7 @@ jobs: $CUDA_VER_ID = "$($CUDA_VER_ARR[0])_$($CUDA_VER_ARR[1])" # Installer url if ( $CUDA_VER_ARR[0] -ge 11 ) { - $CUDA_URL = "http://developer.download.nvidia.com/compute/cuda/$CUDA_VER_FULL/network_installers/cuda_$($CUDA_VER_FULL)_win10_network.exe" + $CUDA_URL = "http://developer.download.nvidia.com/compute/cuda/$CUDA_VER_FULL/network_installers/cuda_$($CUDA_VER_FULL)_windows_network.exe" } else { $CUDA_URL = "http://developer.download.nvidia.com/compute/cuda/$CUDA_VER/Prod/network_installers/cuda_$($CUDA_VER_FULL)_win10_network.exe" } From 164a01a8695a4bee55b10d8f005541dae3510317 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 20 Aug 2024 13:54:05 -0700 Subject: [PATCH 13/26] Update Windows MSVC to CUDA 11.8 Switch tensorflow tests to Py3.12, since TF>2.13.2 not available for Py 3.8 Update stdgpu to try fix thrust-cmake issue CLearer ASSIMP error messages --- .github/workflows/macos.yml | 6 ++++-- .github/workflows/windows.yml | 4 ++-- 3rdparty/README.md | 9 ++------- 3rdparty/find_dependencies.cmake | 1 + 3rdparty/stdgpu/stdgpu.cmake | 3 ++- CMakeLists.txt | 7 +++++++ cpp/open3d/io/file_format/FileASSIMP.cpp | 4 ++-- cpp/open3d/t/io/file_format/FileASSIMP.cpp | 6 +++--- docker/docker_build.sh | 8 ++++---- python/test/t/io/test_realsense.py | 4 +--- 10 files changed, 28 insertions(+), 24 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index ddda24721d3..361891e9f81 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -62,7 +62,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Install dependencies run: | @@ -312,12 +312,14 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.10', '3.11'] + python_version: ['3.10', '3.11', '3.12'] is_main: - ${{ github.ref == 'refs/heads/main' }} exclude: - is_main: false python_version: '3.10' + - is_main: false + python_version: '3.11' steps: - name: Checkout source code # for gh release upload uses: actions/checkout@v4 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 807bc5be5be..57faf093a81 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -25,6 +25,7 @@ env: STOOLS_VER: "67.3.2" JEDI_VER: "0.17.2" # https://github.com/ipython/ipython/issues/12740 IDNA_VER: "2.8" # https://github.com/psf/requests/issues/5710 + CUDA_VERSION: "11.8.0" SRC_DIR: "D:\\a\\open3d\\open3d" BUILD_DIR: "C:\\Open3D\\build" NPROC: 2 @@ -47,7 +48,6 @@ jobs: STATIC_RUNTIME: ON include: - BUILD_CUDA_MODULE: ON - CUDA_VERSION: 11.8.0 env: BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} @@ -64,7 +64,7 @@ jobs: if: ${{ matrix.BUILD_CUDA_MODULE == 'ON' }} run: | # Define variables - $CUDA_VER_FULL = "${{ matrix.CUDA_VERSION }}" + $CUDA_VER_FULL = "${{ env.CUDA_VERSION }}" $CUDA_VER_ARR = $CUDA_VER_FULL.Split(".") $CUDA_VER = "$($CUDA_VER_ARR[0]).$($CUDA_VER_ARR[1])" $CUDA_VER_ID = "$($CUDA_VER_ARR[0])_$($CUDA_VER_ARR[1])" diff --git a/3rdparty/README.md b/3rdparty/README.md index c7c9f94a74e..b5c915fc79b 100644 --- a/3rdparty/README.md +++ b/3rdparty/README.md @@ -68,7 +68,7 @@ tinyobjloader v1.0.0 MIT license Tiny but powerful single file wavefront obj loader https://github.com/syoyo/tinyobjloader -------------------------------------------------------------------------------- -pybind11 v2.6.2 BSD license +pybind11 v2.13.1 BSD license Python binding for C++11 https://github.com/pybind/pybind11 -------------------------------------------------------------------------------- @@ -76,11 +76,6 @@ PoissonReco 12.0 BSD license Poisson Surface Reconstruction https://github.com/mkazhdan/PoissonRecon -------------------------------------------------------------------------------- -Parallel STL 20190522 Apache-2 license -An implementation of the C++ standard library algorithms with support for -execution policies -https://github.com/oneapi-src/oneDPL --------------------------------------------------------------------------------- CUB 1.8.0 BSD license A flexible library of cooperative threadblock primitives and other utilities for CUDA kernel programming @@ -113,7 +108,7 @@ As an alternative, you can modify 3rdparty/zeromq/zeromq_build.cmake to fetch zeromq from our fork https://github.com/isl-org/libzmq -------------------------------------------------------------------------------- -embree 3.13.0 Apache-2 license +embree 4.3.1 Apache-2 license Embree is a collection of high-performance ray tracing kernels https://github.com/embree/embree -------------------------------------------------------------------------------- diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index f3f68698bd3..eb47efb68a2 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -168,6 +168,7 @@ set(ExternalProject_CMAKE_ARGS -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER} -DCMAKE_CUDA_COMPILER_LAUNCHER=${CMAKE_CUDA_COMPILER_LAUNCHER} -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} + -DCMAKE_CUDA_FLAGS=${CMAKE_CUDA_FLAGS} -DCMAKE_SYSTEM_VERSION=${CMAKE_SYSTEM_VERSION} -DCMAKE_INSTALL_LIBDIR=${Open3D_INSTALL_LIB_DIR} # Always build 3rd party code in Release mode. Ignored by multi-config diff --git a/3rdparty/stdgpu/stdgpu.cmake b/3rdparty/stdgpu/stdgpu.cmake index f486fc3d4ce..8a543d11a33 100644 --- a/3rdparty/stdgpu/stdgpu.cmake +++ b/3rdparty/stdgpu/stdgpu.cmake @@ -7,7 +7,8 @@ include(ExternalProject) ExternalProject_Add( ext_stdgpu PREFIX stdgpu - URL https://github.com/stotko/stdgpu/archive/e10f6f3ccc9902d693af4380c3bcd188ec34a2e6.tar.gz + # Aug 2024 + URL https://github.com/stotko/stdgpu/archive/d57a4905f64fea0e06c56194131ad7a1c7291a16.tar.gz URL_HASH SHA256=7bb2733b099f7cedc86d2aee7830d128ac1222cfafa34cbaa4e818483c0a93f6 DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/stdgpu" UPDATE_COMMAND "" diff --git a/CMakeLists.txt b/CMakeLists.txt index 08f4252ba2f..1d5a7ca509e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -402,6 +402,13 @@ cmake_language(EVAL CODE "cmake_language(DEFER CALL open3d_patch_findthreads_mod # Build CUDA module by default if CUDA is available if(BUILD_CUDA_MODULE) + # Suppress nvcc unsupported compiler error for MSVC 2022 with CUDA 11.7 to 12.4 + # https://forums.developer.nvidia.com/t/problems-with-latest-vs2022-update/294150/12 + if (MSVC AND MSVC_VERSION VERSION_LESS_EQUAL "1949") + # Set this before any CUDA checks + set(CMAKE_CUDA_FLAGS "--allow-unsupported-compiler" CACHE STRING "Additional flags for nvcc" FORCE) + message(WARNING "Using --allow-unsupported-compiler flag for nvcc with MSVC 2022.") + endif() if(BUILD_COMMON_CUDA_ARCHS) if (CMAKE_CUDA_ARCHITECTURES) message(STATUS "Building with user-provided architectures: ${CMAKE_CUDA_ARCHITECTURES}") diff --git a/cpp/open3d/io/file_format/FileASSIMP.cpp b/cpp/open3d/io/file_format/FileASSIMP.cpp index 92a2fda8077..a797b8aa1d3 100644 --- a/cpp/open3d/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/io/file_format/FileASSIMP.cpp @@ -170,7 +170,7 @@ bool ReadTriangleMeshUsingASSIMP( const auto* scene = importer.ReadFile(filename.c_str(), post_process_flags); if (!scene) { - utility::LogWarning("Unable to load file {} with ASSIMP", filename); + utility::LogWarning("ReadTriangleMeshUsingASSIMP error: {}", importer.GetErrorString()); return false; } @@ -326,7 +326,7 @@ bool ReadModelUsingAssimp(const std::string& filename, const auto* scene = importer.ReadFile(filename.c_str(), kPostProcessFlags_fast); if (!scene) { - utility::LogWarning("Unable to load file {} with ASSIMP", filename); + utility::LogWarning("ReadTriangleMeshUsingASSIMP error: {}", importer.GetErrorString()); return false; } diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 514d59fca1b..90378622621 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -58,7 +58,7 @@ bool ReadTriangleMeshUsingASSIMP( const auto* scene = importer.ReadFile(filename.c_str(), post_process_flags); if (!scene) { - utility::LogWarning("Unable to load file {} with ASSIMP", filename); + utility::LogWarning("ReadTriangleMeshUsingASSIMP error: {}", importer.GetErrorString()); return false; } @@ -242,7 +242,7 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, if (mesh.HasTriangleColors()) { utility::LogWarning( "Exporting triangle colors is not supported. Please convert to " - "vertex colors or export to a format that supporst it."); + "vertex colors or export to a format that supports it."); } Assimp::Exporter exporter; @@ -470,7 +470,7 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, // Export if (exporter.Export(ai_scene.get(), "glb2", filename.c_str()) == AI_FAILURE) { - utility::LogWarning("Got error: {}", exporter.GetErrorString()); + utility::LogWarning("WriteTriangleMeshUsingASSIMP error: {}", exporter.GetErrorString()); return false; } diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 745473b8e53..c4be56e9e6d 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -322,7 +322,7 @@ ci_build() { export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-4-shared-focal - export PYTHON_VERSION=3.8 + export PYTHON_VERSION=3.12 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi @@ -338,7 +338,7 @@ ci_build() { export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-4-shared-focal - export PYTHON_VERSION=3.8 + export PYTHON_VERSION=3.12 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi @@ -385,7 +385,7 @@ cpu-shared_export_env() { export BASE_IMAGE=ubuntu:20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.8 + export PYTHON_VERSION=3.12 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi @@ -417,7 +417,7 @@ cpu-shared-release_export_env() { export BASE_IMAGE=ubuntu:20.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.8 + export PYTHON_VERSION=3.12 # no TF versions after 2.13.2 for Python 3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF # TODO: tensorflow tests moved here till PyTorch supports cxx11_abi diff --git a/python/test/t/io/test_realsense.py b/python/test/t/io/test_realsense.py index 2d05e1c28d7..c6df67ae181 100755 --- a/python/test/t/io/test_realsense.py +++ b/python/test/t/io/test_realsense.py @@ -64,7 +64,7 @@ def test_RSBagReader(): assert n_frames == 6 # save_frames - bag_reader = o3d.t.io.RGBDVideoReader.create("L515_test_s.bag") + bag_reader = o3d.t.io.RGBDVideoReader.create(sample_l515_bag.path) bag_reader.save_frames("L515_test_s") # Use issubset() since there may be other OS files present assert {'depth', 'color', @@ -79,8 +79,6 @@ def test_RSBagReader(): }.issubset(os.listdir('L515_test_s/color')) shutil.rmtree("L515_test_s") - if os.name != 'nt': # Permission error in Windows - os.remove("L515_test_s.bag") # Test recording from a RealSense camera, if one is connected From 945e35dab2a85bd0e1019b74aea0352c5acc1322 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 22 Aug 2024 16:36:01 -0700 Subject: [PATCH 14/26] Always obey CMAKE_CUDA_ARCHITECTURES if used. Downgrade stdgpu to Jun 19 2024 version due to stdgpu::pair error run CI with ss/python-3.12 branch --- .github/workflows/macos.yml | 1 + .github/workflows/ubuntu-wheel.yml | 1 + .github/workflows/windows.yml | 1 + 3rdparty/stdgpu/stdgpu.cmake | 7 ++++--- CMakeLists.txt | 22 +++++++++++----------- docker/Dockerfile.ci | 3 ++- docker/Dockerfile.wheel | 3 ++- util/ci_utils.sh | 3 +-- 8 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 361891e9f81..1cc7bdb35f3 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -408,6 +408,7 @@ jobs: uses: actions/checkout@v4 with: repository: isl-org/Open3D-ML + ref: ss/python-3.12 # remove before merge path: ${{ env.OPEN3D_ML_ROOT }} - name: Download wheels diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index 411fe6a37bb..c55bc1d65c6 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -153,6 +153,7 @@ jobs: uses: actions/checkout@v4 with: repository: isl-org/Open3D-ML + ref: ss/python-3.12 # remove before merge path: ${{ env.OPEN3D_ML_ROOT }} - name: Download wheels uses: actions/download-artifact@v4 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index b7b2226cb85..9c8d812c55f 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -110,6 +110,7 @@ jobs: if (${env:DEVELOPER_BUILD} -ne "OFF") { ${env:DEVELOPER_BUILD}="ON" } + cmake --version cmake -G "Visual Studio 16 2019" -A x64 ` -DDEVELOPER_BUILD="${env:DEVELOPER_BUILD}" ` -DCMAKE_SYSTEM_VERSION="10.0.19041.0" ` diff --git a/3rdparty/stdgpu/stdgpu.cmake b/3rdparty/stdgpu/stdgpu.cmake index 4ee85e32730..fb9a64e09fd 100644 --- a/3rdparty/stdgpu/stdgpu.cmake +++ b/3rdparty/stdgpu/stdgpu.cmake @@ -7,9 +7,9 @@ include(ExternalProject) ExternalProject_Add( ext_stdgpu PREFIX stdgpu - # Aug 2024 - URL https://github.com/stotko/stdgpu/archive/d57a4905f64fea0e06c56194131ad7a1c7291a16.tar.gz - URL_HASH SHA256=c96775aafb44f5a36af96a4b7ec8b2de6b0587c0cf9ed76705306b6dc3b950b5 + # Jun 20 2024. Later versions need CUDA 11.5 and an API update (stdgpu::pair) + URL https://github.com/stotko/stdgpu/archive/1b6a3319f1fbf180166e1bbc1d75f69ab622a0a0.tar.gz + URL_HASH SHA256=faa3bf9cbe49ef9cc09e2e07e60d10bbf3b896edb6089c920bebe0f850fd95e4 DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/stdgpu" UPDATE_COMMAND "" CMAKE_ARGS @@ -18,6 +18,7 @@ ExternalProject_Add( -DSTDGPU_BUILD_SHARED_LIBS=OFF -DSTDGPU_BUILD_EXAMPLES=OFF -DSTDGPU_BUILD_TESTS=OFF + -DSTDGPU_BUILD_BENCHMARKS=OFF -DSTDGPU_ENABLE_CONTRACT_CHECKS=OFF -DTHRUST_INCLUDE_DIR=${CUDAToolkit_INCLUDE_DIRS} ${ExternalProject_CMAKE_ARGS_hidden} diff --git a/CMakeLists.txt b/CMakeLists.txt index a9d0e3b0ee1..f781ea2e708 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -394,10 +394,10 @@ if(BUILD_CUDA_MODULE) set(CMAKE_CUDA_FLAGS "--allow-unsupported-compiler" CACHE STRING "Additional flags for nvcc" FORCE) message(WARNING "Using --allow-unsupported-compiler flag for nvcc with MSVC 2022.") endif() - if(BUILD_COMMON_CUDA_ARCHS) - if (CMAKE_CUDA_ARCHITECTURES) - message(STATUS "Building with user-provided architectures: ${CMAKE_CUDA_ARCHITECTURES}") - else() + if (CMAKE_CUDA_ARCHITECTURES) + message(STATUS "Building with user-provided CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") + else() + if(BUILD_COMMON_CUDA_ARCHS) # Build with all supported architectures for previous 2 generations and # M0 (minor=0) architectures for previous generations (including # deprecated). Note that cubin for M0 runs on GPUs with architecture Mx. @@ -417,14 +417,14 @@ if(BUILD_CUDA_MODULE) set(CMAKE_CUDA_ARCHITECTURES 30-real 50-real 60-real 70-real 75) # Kepler, Maxwell, Pascal, Turing endif() message(STATUS "Using CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") - endif() - else() - execute_process(COMMAND nvidia-smi RESULT_VARIABLE NVIDIA_CHECK OUTPUT_QUIET) - if (NVIDIA_CHECK EQUAL 0) - message(STATUS "Building with native CUDA architecture.") - set(CMAKE_CUDA_ARCHITECTURES native) else() - message(WARNING "No CUDA GPU detected. Building with CMake default CUDA architecture.") + execute_process(COMMAND nvidia-smi RESULT_VARIABLE NVIDIA_CHECK OUTPUT_QUIET) + if (NVIDIA_CHECK EQUAL 0) + message(STATUS "Building with native CUDA architecture.") + set(CMAKE_CUDA_ARCHITECTURES native) + else() + message(WARNING "No CUDA GPU detected. Building with CMake default CUDA architecture.") + endif() endif() endif() enable_language(CUDA) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index cd52a838882..2c0daa949f1 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -139,7 +139,8 @@ RUN CCACHE_DIR=$(ccache -p | grep cache_dir | grep -oE "[^ ]+$") \ # Checkout Open3D-ML main branch # TODO: We may add support for local Open3D-ML repo or pinned ML repo tag ENV OPEN3D_ML_ROOT=/root/Open3D-ML -RUN git clone --depth 1 https://github.com/isl-org/Open3D-ML.git ${OPEN3D_ML_ROOT} +# Remove branch before merge +RUN git clone --depth 1 --branch ss/python-3.12 https://github.com/isl-org/Open3D-ML.git ${OPEN3D_ML_ROOT} # Open3D repo # Always keep /root/Open3D as the WORKDIR diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index 3bcfe4a6a94..9d86856c177 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -98,7 +98,8 @@ RUN which python \ # Checkout Open3D-ML main branch # TODO: We may add support for local Open3D-ML repo or pinned ML repo tag ENV OPEN3D_ML_ROOT=/root/Open3D-ML -RUN git clone https://github.com/isl-org/Open3D-ML.git ${OPEN3D_ML_ROOT} +# Remove branch before merge +RUN git clone --depth 1 --branch ss/python-3.12 https://github.com/isl-org/Open3D-ML.git ${OPEN3D_ML_ROOT} # Open3D C++ dependencies # Done before copying the full Open3D directory for better Docker caching diff --git a/util/ci_utils.sh b/util/ci_utils.sh index a3785ea8582..4d1b96faba9 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -365,8 +365,7 @@ install_docs_dependencies() { echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}" python -m pip install -r "${OPEN3D_ML_ROOT}/requirements.txt" python -m pip install -r "${OPEN3D_ML_ROOT}/requirements-torch.txt" - python -m pip install -r "${OPEN3D_ML_ROOT}/requirements-tensorflow.txt" || - python -m pip install tensorflow # FIXME: Remove after Open3D-ML update + python -m pip install -r "${OPEN3D_ML_ROOT}/requirements-tensorflow.txt" else echo OPEN3D_ML_ROOT="$OPEN3D_ML_ROOT" not specified or invalid. Skipping ML dependencies. fi From 2222fcb87b5e5ab6146070b3f84e00eeaa763f5a Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Wed, 28 Aug 2024 14:37:03 -0700 Subject: [PATCH 15/26] use shared_ptr to capture array in lambda --- cpp/pybind/core/tensor_converter.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/pybind/core/tensor_converter.cpp b/cpp/pybind/core/tensor_converter.cpp index 018d857e5ed..1e07ad91d0c 100644 --- a/cpp/pybind/core/tensor_converter.cpp +++ b/cpp/pybind/core/tensor_converter.cpp @@ -106,10 +106,10 @@ Tensor PyArrayToTensor(py::array array, bool inplace) { Dtype dtype = pybind_utils::ArrayFormatToDtype(info.format, info.itemsize); Device device("CPU:0"); - array.inc_ref(); - std::function deleter = [array](void*) -> void { + auto shared_array = std::make_shared(array); + std::function deleter = [shared_array](void*) mutable -> void { py::gil_scoped_acquire acquire; - array.dec_ref(); + shared_array.reset(); }; auto blob = std::make_shared(device, info.ptr, deleter); Tensor t_inplace(shape, strides, info.ptr, dtype, blob); From 35ee51e10064c2a897dea6c18bf124aac9a24db3 Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Sun, 1 Sep 2024 20:57:41 +0200 Subject: [PATCH 16/26] fix tf ml ops tests on macos --- python/open3d/ml/tf/python/layers/neighbor_search.py | 9 +++++++-- python/test/ml_ops/test_sparseconv.py | 8 ++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/python/open3d/ml/tf/python/layers/neighbor_search.py b/python/open3d/ml/tf/python/layers/neighbor_search.py index 8a9059ef485..d66b28fcf32 100644 --- a/python/open3d/ml/tf/python/layers/neighbor_search.py +++ b/python/open3d/ml/tf/python/layers/neighbor_search.py @@ -121,6 +121,11 @@ def call(self, queries_row_splits = queries.row_splits queries = queries.values + if isinstance(radius, tf.Tensor): + radius_ = tf.cast(radius, points.dtype) + else: + radius_ = radius + if points_row_splits is None: points_row_splits = tf.cast(tf.stack([0, tf.shape(points)[0]]), dtype=tf.int64) @@ -131,7 +136,7 @@ def call(self, table = ops.build_spatial_hash_table( max_hash_table_size=self.max_hash_table_size, points=points, - radius=radius, + radius=radius_, points_row_splits=points_row_splits, hash_table_size_factor=hash_table_size_factor) else: @@ -142,7 +147,7 @@ def call(self, metric=self.metric, points=points, queries=queries, - radius=radius, + radius=radius_, points_row_splits=points_row_splits, queries_row_splits=queries_row_splits, hash_table_splits=table.hash_table_splits, diff --git a/python/test/ml_ops/test_sparseconv.py b/python/test/ml_ops/test_sparseconv.py index 1fd6ed5058b..93bd8b53f3a 100644 --- a/python/test/ml_ops/test_sparseconv.py +++ b/python/test/ml_ops/test_sparseconv.py @@ -91,7 +91,7 @@ def bias_initializer(a): y = mltest.run_op(ml, ml.device, True, sparse_conv, inp_features, inp_positions * voxel_size, out_positions * voxel_size, - voxel_size, inp_importance) + voxel_size=voxel_size, inp_importance=inp_importance) # Compare the output to a standard 3d conv # store features in a volume to use standard 3d convs @@ -212,7 +212,7 @@ def test_compare_to_conv3d_batches(ml, dtype, kernel_size, out_channels, y = mltest.run_op(ml, ml.device, True, sparse_conv, inp_features, inp_positions * voxel_size, out_positions * voxel_size, - voxel_size, inp_importance) + voxel_size=voxel_size, inp_importance=inp_importance) for idx in range(batch_size): inp_pos = inp_positions[idx].numpy() inp_feat = inp_features[idx].numpy() @@ -338,7 +338,7 @@ def bias_initializer(a): y = mltest.run_op(ml, ml.device, True, sparse_conv_transpose, inp_features, inp_positions * voxel_size, out_positions * voxel_size, - voxel_size, out_importance) + voxel_size=voxel_size, out_importance=out_importance) # Compare the output to a standard 3d conv # store features in a volume to use standard 3d convs @@ -465,7 +465,7 @@ def test_compare_to_conv3dtranspose_batches(ml, dtype, kernel_size, y = mltest.run_op(ml, ml.device, True, sparse_conv_transpose, inp_features, inp_positions * voxel_size, out_positions * voxel_size, - voxel_size, out_importance) + voxel_size=voxel_size, out_importance=out_importance) for idx in range(batch_size): inp_pos = inp_positions[idx].numpy() inp_feat = inp_features[idx].numpy() From 170b65027244f6fa431602628d6c96b030b7b6e6 Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Wed, 4 Sep 2024 09:41:21 +0200 Subject: [PATCH 17/26] apply style --- .../ml/tf/python/layers/neighbor_search.py | 4 +- python/test/ml_ops/test_sparseconv.py | 48 ++++++++++++++----- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/python/open3d/ml/tf/python/layers/neighbor_search.py b/python/open3d/ml/tf/python/layers/neighbor_search.py index d66b28fcf32..d13dd799d11 100644 --- a/python/open3d/ml/tf/python/layers/neighbor_search.py +++ b/python/open3d/ml/tf/python/layers/neighbor_search.py @@ -122,9 +122,9 @@ def call(self, queries = queries.values if isinstance(radius, tf.Tensor): - radius_ = tf.cast(radius, points.dtype) + radius_ = tf.cast(radius, points.dtype) else: - radius_ = radius + radius_ = radius if points_row_splits is None: points_row_splits = tf.cast(tf.stack([0, tf.shape(points)[0]]), diff --git a/python/test/ml_ops/test_sparseconv.py b/python/test/ml_ops/test_sparseconv.py index 93bd8b53f3a..71a019ccc50 100644 --- a/python/test/ml_ops/test_sparseconv.py +++ b/python/test/ml_ops/test_sparseconv.py @@ -89,9 +89,15 @@ def bias_initializer(a): if ml.module.__name__ == 'torch': sparse_conv.to(ml.device) - y = mltest.run_op(ml, ml.device, True, sparse_conv, inp_features, - inp_positions * voxel_size, out_positions * voxel_size, - voxel_size=voxel_size, inp_importance=inp_importance) + y = mltest.run_op(ml, + ml.device, + True, + sparse_conv, + inp_features, + inp_positions * voxel_size, + out_positions * voxel_size, + voxel_size=voxel_size, + inp_importance=inp_importance) # Compare the output to a standard 3d conv # store features in a volume to use standard 3d convs @@ -210,9 +216,15 @@ def test_compare_to_conv3d_batches(ml, dtype, kernel_size, out_channels, inp_features = tf.RaggedTensor.from_row_splits( values=inp_features, row_splits=inp_positions_row_splits) - y = mltest.run_op(ml, ml.device, True, sparse_conv, inp_features, - inp_positions * voxel_size, out_positions * voxel_size, - voxel_size=voxel_size, inp_importance=inp_importance) + y = mltest.run_op(ml, + ml.device, + True, + sparse_conv, + inp_features, + inp_positions * voxel_size, + out_positions * voxel_size, + voxel_size=voxel_size, + inp_importance=inp_importance) for idx in range(batch_size): inp_pos = inp_positions[idx].numpy() inp_feat = inp_features[idx].numpy() @@ -336,9 +348,15 @@ def bias_initializer(a): if ml.module.__name__ == 'torch': sparse_conv_transpose.to(ml.device) - y = mltest.run_op(ml, ml.device, True, sparse_conv_transpose, inp_features, - inp_positions * voxel_size, out_positions * voxel_size, - voxel_size=voxel_size, out_importance=out_importance) + y = mltest.run_op(ml, + ml.device, + True, + sparse_conv_transpose, + inp_features, + inp_positions * voxel_size, + out_positions * voxel_size, + voxel_size=voxel_size, + out_importance=out_importance) # Compare the output to a standard 3d conv # store features in a volume to use standard 3d convs @@ -463,9 +481,15 @@ def test_compare_to_conv3dtranspose_batches(ml, dtype, kernel_size, inp_features = tf.RaggedTensor.from_row_splits( values=inp_features, row_splits=inp_positions_row_splits) - y = mltest.run_op(ml, ml.device, True, sparse_conv_transpose, inp_features, - inp_positions * voxel_size, out_positions * voxel_size, - voxel_size=voxel_size, out_importance=out_importance) + y = mltest.run_op(ml, + ml.device, + True, + sparse_conv_transpose, + inp_features, + inp_positions * voxel_size, + out_positions * voxel_size, + voxel_size=voxel_size, + out_importance=out_importance) for idx in range(batch_size): inp_pos = inp_positions[idx].numpy() inp_feat = inp_features[idx].numpy() From 68fca39296d04cd6ed68beb71de5f30ba4fffafb Mon Sep 17 00:00:00 2001 From: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:20:54 -0700 Subject: [PATCH 18/26] Install thrust with CUDA --- .github/workflows/windows.yml | 2 +- CMakeLists.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 9c8d812c55f..e08bd1960c0 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -77,7 +77,7 @@ jobs: # Installer arguments $CUDA_INSTALL_ARGS = "-s" # Required packages - $CUDA_PACKAGES = "nvcc", "visual_studio_integration", "cublas", "cublas_dev", "cudart", "cusolver", "cusolver_dev", "npp", "npp_dev" + $CUDA_PACKAGES = "nvcc", "visual_studio_integration", "cublas", "cublas_dev", "cudart", "cusolver", "cusolver_dev", "npp", "npp_dev", "thrust" $CUDA_PACKAGES.ForEach({ $CUDA_INSTALL_ARGS += " $($_)_$($CUDA_VER)" }) # Download and install CUDA echo "Downloading CUDA installer from $CUDA_URL" diff --git a/CMakeLists.txt b/CMakeLists.txt index f781ea2e708..7b047cd0b82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -392,7 +392,8 @@ if(BUILD_CUDA_MODULE) if (MSVC AND MSVC_VERSION VERSION_LESS_EQUAL "1949") # Set this before any CUDA checks set(CMAKE_CUDA_FLAGS "--allow-unsupported-compiler" CACHE STRING "Additional flags for nvcc" FORCE) - message(WARNING "Using --allow-unsupported-compiler flag for nvcc with MSVC 2022.") + message(WARNING "Using --allow-unsupported-compiler flag for nvcc with MSVC 2022. " + "Set $Env:NVCC_PREPEND_FLAGS='--allow-unsupported-compiler' if nvcc still fails.") endif() if (CMAKE_CUDA_ARCHITECTURES) message(STATUS "Building with user-provided CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") From 631404639883639fac870cae55fdc08009b34074 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 12 Sep 2024 17:10:21 -0700 Subject: [PATCH 19/26] Remove fmt formatter for embree RTCError to fix macOS error --- cpp/open3d/t/geometry/RaycastingScene.cpp | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/cpp/open3d/t/geometry/RaycastingScene.cpp b/cpp/open3d/t/geometry/RaycastingScene.cpp index 31fdbeef856..901bc651ed0 100644 --- a/cpp/open3d/t/geometry/RaycastingScene.cpp +++ b/cpp/open3d/t/geometry/RaycastingScene.cpp @@ -33,7 +33,8 @@ typedef Eigen::Vector3f Vec3f; // Error function called by embree. void ErrorFunction(void* userPtr, enum RTCError error, const char* str) { - open3d::utility::LogError("embree error: {} {}", error, str); + open3d::utility::LogError("Embree error: {} {}", rtcGetErrorString(error), + str); } // Checks the last dim, ensures that the number of dims is >= min_ndim, checks @@ -1173,20 +1174,4 @@ uint32_t RaycastingScene::INVALID_ID() { return RTC_INVALID_GEOMETRY_ID; } } // namespace geometry } // namespace t -} // namespace open3d - -namespace fmt { -template <> -struct formatter { - template - auto format(const RTCError& c, FormatContext& ctx) { - const char* name = rtcGetErrorString(c); - return format_to(ctx.out(), name); - } - - template - constexpr auto parse(ParseContext& ctx) -> decltype(ctx.begin()) { - return ctx.begin(); - } -}; -} // namespace fmt +} // namespace open3d \ No newline at end of file From 99bc91fd8f635b88564f35740450d8ac5198211f Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Sun, 15 Sep 2024 11:30:52 -0700 Subject: [PATCH 20/26] switch to miniforge --- docker/Dockerfile.ci | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 2c0daa949f1..d910927c2cb 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -76,18 +76,19 @@ RUN apt-get update && apt-get install -y \ pkg-config \ && rm -rf /var/lib/apt/lists/* -# Miniconda or Intel conda +# Miniforge or Intel conda # The **/open3d/bin paths are used during docker run, in this way docker run # does not need to activate the environment again. -ENV PATH="/root/miniconda3/bin:${PATH}" -ENV PATH="/root/miniconda3/envs/open3d/bin:${PATH}" +ENV PATH="/root/miniforge3/bin:${PATH}" +ENV PATH="/root/miniforge3/envs/open3d/bin:${PATH}" ENV PATH="/opt/intel/oneapi/intelpython/latest/bin:${PATH}" ENV PATH="/opt/intel/oneapi/intelpython/latest/envs/open3d/bin:${PATH}" RUN if [ "${BUILD_SYCL_MODULE}" = "OFF" ]; then \ - wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh; \ - bash Miniconda3-latest-Linux-x86_64.sh -b; \ - rm Miniconda3-latest-Linux-x86_64.sh; \ + wget "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"; \ + bash Miniforge3-$(uname)-$(uname -m).sh -b; \ + rm Miniforge3-$(uname)-$(uname -m).sh; \ fi + RUN conda --version \ && conda create -y -n open3d python=${PYTHON_VERSION} From 7cdd7e337bfeaca2fc690eace7fda7b942cc39ea Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Sun, 15 Sep 2024 11:31:51 -0700 Subject: [PATCH 21/26] use cuda 12.3 as suggested here https://github.com/tensorflow/tensorflow/issues/63356 --- docker/docker_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker_build.sh b/docker/docker_build.sh index c4be56e9e6d..48cae885419 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -84,8 +84,8 @@ HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pw CCACHE_VERSION=4.3 CMAKE_VERSION=cmake-3.24.4-linux-x86_64 CMAKE_VERSION_AARCH64=cmake-3.24.4-linux-aarch64 -CUDA_VERSION=11.8.0-cudnn8 -CUDA_VERSION_LATEST=11.8.0-cudnn8 +CUDA_VERSION=12.3.2-cudnn9 +CUDA_VERSION_LATEST=12.3.2-cudnn9 print_usage_and_exit_docker_build() { echo "$__usage_docker_build" From 13699e8cb10357199b99ad7f351e7981da739414 Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Mon, 16 Sep 2024 15:31:59 -0700 Subject: [PATCH 22/26] change to cuda 12.1 for pytorch and tensorflow --- docker/docker_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 48cae885419..c92aed26829 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -84,8 +84,8 @@ HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pw CCACHE_VERSION=4.3 CMAKE_VERSION=cmake-3.24.4-linux-x86_64 CMAKE_VERSION_AARCH64=cmake-3.24.4-linux-aarch64 -CUDA_VERSION=12.3.2-cudnn9 -CUDA_VERSION_LATEST=12.3.2-cudnn9 +CUDA_VERSION=12.1.0-cudnn8 +CUDA_VERSION_LATEST=12.1.0-cudnn8 print_usage_and_exit_docker_build() { echo "$__usage_docker_build" From 267df1aa47984a489576754f6dc015a213530a98 Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Tue, 17 Sep 2024 01:41:59 -0700 Subject: [PATCH 23/26] set TORCH_CUDA_ARCH_LIST as workaround for CUDA 12 and Pytorch <2.4 --- docker/Dockerfile.ci | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index d910927c2cb..edca2aedc54 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -183,7 +183,10 @@ RUN \ export CMAKE_CXX_COMPILER=g++; \ export CMAKE_C_COMPILER=gcc; \ # TODO: PyTorch still use old CXX ABI, remove this line when PyTorch is updated + # TODO: Using CUDA 12.x and Pytorch <2.4 gives the error "Unknown CUDA Architecture Name 9.0a in CUDA_SELECT_NVCC_ARCH_FLAGS". + # As a workaround we explicitly set TORCH_CUDA_ARCH_LIST if [ "$BUILD_PYTORCH_OPS" = "ON" ]; then \ + export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" \ export GLIBCXX_USE_CXX11_ABI=OFF; \ else \ export GLIBCXX_USE_CXX11_ABI=ON; \ From cc015124b5d3396244fce9ffb7795c87ffc19777 Mon Sep 17 00:00:00 2001 From: Benjamin Ummenhofer Date: Thu, 19 Sep 2024 01:40:11 -0700 Subject: [PATCH 24/26] fix failed merge --- cpp/open3d/t/geometry/RaycastingScene.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/open3d/t/geometry/RaycastingScene.cpp b/cpp/open3d/t/geometry/RaycastingScene.cpp index 4a4482ebc4d..12f082d0ee1 100644 --- a/cpp/open3d/t/geometry/RaycastingScene.cpp +++ b/cpp/open3d/t/geometry/RaycastingScene.cpp @@ -1174,7 +1174,7 @@ uint32_t RaycastingScene::INVALID_ID() { return RTC_INVALID_GEOMETRY_ID; } } // namespace geometry } // namespace t - +} // namespace open3d namespace fmt { template <> @@ -1191,4 +1191,3 @@ struct formatter { } }; } // namespace fmt - From 4ccae4217bffe7fcbb18716a39e7d120c3c537e0 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 3 Oct 2024 00:05:24 -0700 Subject: [PATCH 25/26] Match PyTorch ops CUDA arch list to that of Open3D --- 3rdparty/cmake/FindPytorch.cmake | 23 +++++++++++++++++++++++ docker/Dockerfile.ci | 3 --- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/3rdparty/cmake/FindPytorch.cmake b/3rdparty/cmake/FindPytorch.cmake index eb2a53e2ec5..28da7ec71ba 100644 --- a/3rdparty/cmake/FindPytorch.cmake +++ b/3rdparty/cmake/FindPytorch.cmake @@ -14,6 +14,20 @@ # # and import the target 'torch'. +# "80-real" to "8.0" and "80" to "8.0+PTX": +macro(translate_arch_string input output) + if("${input}" MATCHES "[0-9]+-real") + string(REGEX REPLACE "([1-9])([0-9])-real" "\\1.\\2" version "${input}") + elseif("${input}" MATCHES "([0-9]+)") + string(REGEX REPLACE "([1-9])([0-9])" "\\1.\\2+PTX" version "${input}") + elseif(input STREQUAL "native") + set(version "Auto") + else() + message(FATAL_ERROR "Invalid architecture string: ${input}") + endif() + set(${output} "${version}") +endmacro() + if(NOT Pytorch_FOUND) # Searching for pytorch requires the python executable if (NOT Python3_EXECUTABLE) @@ -41,6 +55,15 @@ if(NOT Pytorch_FOUND) unset(PyTorch_FETCH_PROPERTIES) unset(PyTorch_PROPERTIES) + # Using CUDA 12.x and Pytorch <2.4 gives the error "Unknown CUDA Architecture Name 9.0a in CUDA_SELECT_NVCC_ARCH_FLAGS". + # As a workaround we explicitly set TORCH_CUDA_ARCH_LIST + set(TORCH_CUDA_ARCH_LIST "") + foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES) + translate_arch_string("${arch}" ptarch) + list(APPEND TORCH_CUDA_ARCH_LIST "${ptarch}") + endforeach() + + message(STATUS "Using top level CMAKE_CUDA_ARCHITECTURES for TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST}") # Use the cmake config provided by torch find_package(Torch REQUIRED PATHS "${Pytorch_ROOT}" NO_DEFAULT_PATH) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index dcdc0599dd9..5c5991e5a76 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -196,10 +196,7 @@ RUN \ export CMAKE_CXX_COMPILER=g++; \ export CMAKE_C_COMPILER=gcc; \ # TODO: PyTorch still use old CXX ABI, remove this line when PyTorch is updated - # TODO: Using CUDA 12.x and Pytorch <2.4 gives the error "Unknown CUDA Architecture Name 9.0a in CUDA_SELECT_NVCC_ARCH_FLAGS". - # As a workaround we explicitly set TORCH_CUDA_ARCH_LIST if [ "$BUILD_PYTORCH_OPS" = "ON" ]; then \ - export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" \ export GLIBCXX_USE_CXX11_ABI=OFF; \ else \ export GLIBCXX_USE_CXX11_ABI=ON; \ From b6cc8321383cb657ac84eb077ad897cb2686b2d7 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 3 Oct 2024 10:32:14 -0700 Subject: [PATCH 26/26] do not install cuda pytorch before build_pip_package --- 3rdparty/cmake/FindPytorch.cmake | 14 ++++++++------ docker/Dockerfile.wheel | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/3rdparty/cmake/FindPytorch.cmake b/3rdparty/cmake/FindPytorch.cmake index 28da7ec71ba..31babdca149 100644 --- a/3rdparty/cmake/FindPytorch.cmake +++ b/3rdparty/cmake/FindPytorch.cmake @@ -55,15 +55,17 @@ if(NOT Pytorch_FOUND) unset(PyTorch_FETCH_PROPERTIES) unset(PyTorch_PROPERTIES) + if(BUILD_CUDA_MODULE) # Using CUDA 12.x and Pytorch <2.4 gives the error "Unknown CUDA Architecture Name 9.0a in CUDA_SELECT_NVCC_ARCH_FLAGS". # As a workaround we explicitly set TORCH_CUDA_ARCH_LIST - set(TORCH_CUDA_ARCH_LIST "") - foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES) - translate_arch_string("${arch}" ptarch) - list(APPEND TORCH_CUDA_ARCH_LIST "${ptarch}") - endforeach() + set(TORCH_CUDA_ARCH_LIST "") + foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES) + translate_arch_string("${arch}" ptarch) + list(APPEND TORCH_CUDA_ARCH_LIST "${ptarch}") + endforeach() + message(STATUS "Using top level CMAKE_CUDA_ARCHITECTURES for TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST}") + endif() - message(STATUS "Using top level CMAKE_CUDA_ARCHITECTURES for TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST}") # Use the cmake config provided by torch find_package(Torch REQUIRED PATHS "${Pytorch_ROOT}" NO_DEFAULT_PATH) diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index 9d86856c177..71a16a2ac13 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -113,7 +113,7 @@ COPY ./python/requirements.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/ RUN source /root/Open3D/util/ci_utils.sh \ - && install_python_dependencies with-cuda with-jupyter + && install_python_dependencies with-jupyter # Open3D Jupyter dependencies RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \