Supporting TECA_assets svn repo in TECA

LBL-EESA · May 12, 2020 · ae8e18f · ae8e18f
1 parent bf7996f
commit ae8e18f
Show file tree

Hide file tree

Showing 14 changed files with 69 additions and 12 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -15,6 +15,7 @@ env:
     - TECA_DIR=/travis_teca_dir
     - TECA_PYTHON_VERSION=3
     - TECA_DATA_REVISION=56
+    - TECA_ASSETS_REVISION=2
   jobs:
     - DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04
     - DOCKER_IMAGE=fedora IMAGE_VERSION=31 IMAGE_NAME=fedora_31
@@ -48,6 +49,7 @@ install:
     docker exec teca_${DOCKER_IMAGE}_${IMAGE_VERSION} /bin/bash -c
     "export TECA_PYTHON_VERSION=${TECA_PYTHON_VERSION} &&
     export TECA_DATA_REVISION=${TECA_DATA_REVISION} &&
+    export TECA_ASSETS_REVISION=${TECA_ASSETS_REVISION} &&
     ${TECA_DIR}/test/travis_ci/install_${IMAGE_NAME}.sh";
     fi
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -327,6 +327,20 @@ else()
 endif()
 set(TECA_HAS_DATA ${tmp} CACHE BOOL "TECA_data is present")
 
+# do we have access to a checkout of TECA_assets?
+set(tmp OFF)
+set(TECA_ASSETS_ROOT "/path/to/TECA_assets"
+    CACHE PATH "Path to TECA_assets repository")
+if (EXISTS "${TECA_ASSETS_ROOT}")
+    set(tmp ON)
+    message(STATUS "TECA_assets -- available")
+elseif (REQUIRE_TECA_ASSETS)
+    message(FATAL_ERROR "TECA_assets -- required but not found")
+else()
+    message(STATUS "TECA_assets -- not available")
+endif()
+set(TECA_HAS_ASSETS ${tmp} CACHE BOOL "TECA_assets is present")
+
 # generate global configuration
 include(CMake/teca_test.cmake)
 include(CMake/teca_app.cmake)

diff --git a/doc/rtd/applications.rst b/doc/rtd/applications.rst
@@ -419,7 +419,7 @@ Example
 .. code-block:: bash
 
     mpiexec -np 10 ./bin/teca_tc_trajectory_scalars   \
-        --texture ../../TECA_data/earthmap4k.png      \
+        --texture ../../TECA_assets/earthmap4k.png      \
         tracks_1990s_3hr_mdd_4800.bin                 \
         traj_scalars_1990s_3hr_mdd_4800
 

diff --git a/doc/teca_users_guide.tex b/doc/teca_users_guide.tex
@@ -251,6 +251,7 @@ \subsubsection{Installing TECA}
     -DCMAKE_INSTALL_PREFIX=${TECA_INSTALL_DIR} \
     -DBUILD_TESTING=ON \
     -DTECA_DATA_ROOT=${TECA_DATA_DIR} \
+    -DTECA_ASSETS_ROOT=${TECA_ASSETS_DIR} \
     ${TECA_SOURCE_DIR}
 
 make -j4 && make -j4 install
@@ -270,6 +271,7 @@ \subsubsection{Installing TECA}
     -DCMAKE_INSTALL_PREFIX=${TECA_INSTALL_DIR} \
     -DBUILD_TESTING=ON \
     -DTECA_DATA_ROOT=${TECA_DATA_DIR} \
+    -DTECA_ASSETS_ROOT=${TECA_ASSETS_DIR} \
     ${TECA_SOURCE_DIR}
 
 make -j4 && make -j4 install
@@ -710,7 +712,7 @@ \subsection{Command Line Arguments}
 \subsection{Example}
 \begin{minted}{bash}
 mpiexec -np 10 ./bin/teca_tc_trajectory_scalars   \
-    --texture ../../TECA_data/earthmap4k.png      \
+    --texture ../../TECA_assets/earthmap4k.png      \
     tracks_1990s_3hr_mdd_4800.bin                 \
     traj_scalars_1990s_3hr_mdd_4800
 \end{minted}

diff --git a/python/teca_py_config.i b/python/teca_py_config.i
@@ -42,6 +42,27 @@ const char *get_teca_data_root()
 #endif
 }
 
+// return true if the location of TECA_assets repo is known
+bool get_teca_has_assets()
+{
+#if defined(TECA_HAS_ASSETS)
+    return true;
+#else
+    return false;
+#endif
+}
+
+// return the path to a TECA_assets repo check out, or None if the
+// path is not known at compile time
+const char *get_teca_assets_root()
+{
+#if defined(TECA_HAS_ASSETS)
+    return TECA_ASSETS_ROOT;
+#else
+    return nullptr;
+#endif
+}
+
 // return true if TECA was compiled with regex support
 bool get_teca_has_regex()
 {

diff --git a/teca_config.h.in b/teca_config.h.in
@@ -15,11 +15,13 @@
 
 #cmakedefine TECA_VERSION_DESCR "@TECA_VERSION_DESCR@"
 #cmakedefine TECA_PYTHON_VERSION @TECA_PYTHON_VERSION@
-#cmakedefine TECA_DATA_ROOT "@TECA_DATA_ROOT@"
 
 #cmakedefine TECA_ENABLE_PROFILER
 
 #cmakedefine TECA_HAS_DATA
 #cmakedefine TECA_DATA_ROOT "@TECA_DATA_ROOT@"
 
+#cmakedefine TECA_HAS_ASSETS
+#cmakedefine TECA_ASSETS_ROOT "@TECA_ASSETS_ROOT@"
+
 #endif
diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt
@@ -185,7 +185,7 @@ teca_add_test(py_test_tc_activity
 teca_add_test(py_test_tc_trajectory_scalars_serial
     COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_tc_trajectory_scalars.py
     "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800_median_in_cat_wr.bin"
-    "${TECA_DATA_ROOT}/earthmap4k.png"
+    "${TECA_ASSETS_ROOT}/earthmap4k.png"
     "${TECA_DATA_ROOT}/py_test_tc_trajectory_scalars.bin" 0 -1
     FEATURES ${TECA_HAS_UDUNITS}
     REQ_TECA_DATA)
@@ -194,7 +194,7 @@ teca_add_test(py_test_tc_trajectory_scalars_mpi
     COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE}
     ${CMAKE_CURRENT_SOURCE_DIR}/test_tc_trajectory_scalars.py
     "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800_median_in_cat_wr.bin"
-    "${TECA_DATA_ROOT}/earthmap4k.png"
+    "${TECA_ASSETS_ROOT}/earthmap4k.png"
     "${TECA_DATA_ROOT}/py_test_tc_trajectory_scalars.bin" 0 -1
     FEATURES ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4Py_FOUND}
     REQ_TECA_DATA)
@@ -232,20 +232,20 @@ teca_add_test(py_test_bayesian_ar_detect_mpi_threads
 
 teca_add_test(py_test_deeplabv3p_ar_detect
     COMMAND  ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py
-    "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt"
-    "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth"
+    "${TECA_ASSETS_ROOT}/cascade_deeplab_IVT.pt"
+    "${TECA_ASSETS_ROOT}/resnet101-5d3b4d8f-state_dict.pth"
     "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$"
     "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT 1
     FEATURES ${TECA_HAS_NETCDF}
     REQ_TECA_DATA)
 
 teca_add_test(py_test_deeplabv3p_ar_detect_mpi
     COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE}
-        ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py
-       "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt"
-        "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth"
-        "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$"
-        "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT 1
+    ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py
+    "${TECA_ASSETS_ROOT}/cascade_deeplab_IVT.pt"
+    "${TECA_ASSETS_ROOT}/resnet101-5d3b4d8f-state_dict.pth"
+    "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$"
+    "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT 1
     FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} ${MPI4Py_FOUND}
     REQ_TECA_DATA)
 

diff --git a/test/python/test_config.py b/test/python/test_config.py
@@ -13,3 +13,5 @@
 sys.stderr.write('TECA_HAS_OPENSSL=%s\n'%(str(get_teca_has_openssl())))
 sys.stderr.write('TECA_HAS_DATA=%s\n'%(str(get_teca_has_data())))
 sys.stderr.write('TECA_DATA_ROOT=%s\n'%(get_teca_data_root()))
+sys.stderr.write('TECA_HAS_ASSETS=%s\n'%(str(get_teca_has_assets())))
+sys.stderr.write('TECA_ASSETS_ROOT=%s\n'%(get_teca_assets_root()))
diff --git a/test/travis_ci/ctest_linux.cmake b/test/travis_ci/ctest_linux.cmake
@@ -20,6 +20,7 @@ BUILD_TESTING=ON
 TECA_ENABLE_PROFILER=ON
 TECA_PYTHON_VERSION=$ENV{TECA_PYTHON_VERSION}
 TECA_DATA_ROOT=$ENV{DASHROOT}/TECA_data
+TECA_ASSETS_ROOT=$ENV{DASHROOT}/TECA_assets
 TECA_TEST_CORES=2
 REQUIRE_OPENSSL=TRUE
 REQUIRE_BOOST=TRUE

diff --git a/test/travis_ci/ctest_osx.cmake b/test/travis_ci/ctest_osx.cmake
@@ -20,6 +20,7 @@ BUILD_TESTING=ON
 TECA_ENABLE_PROFILER=ON
 TECA_PYTHON_VERSION=$ENV{TECA_PYTHON_VERSION}
 TECA_DATA_ROOT=$ENV{DASHROOT}/TECA_data
+TECA_ASSETS_ROOT=$ENV{DASHROOT}/TECA_assets
 TECA_TEST_CORES=2
 REQUIRE_OPENSSL=TRUE
 OPENSSL_ROOT_DIR=/usr/local/opt/[email protected]

diff --git a/test/travis_ci/install_fedora_31.sh b/test/travis_ci/install_fedora_31.sh
@@ -31,3 +31,4 @@ pip3 install numpy mpi4py matplotlib torch
 
 # install data files.
 svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data
+svn co svn://svn.code.sf.net/p/teca/TECA_assets@${TECA_ASSETS_REVISION} TECA_assets
diff --git a/test/travis_ci/install_osx.sh b/test/travis_ci/install_osx.sh
@@ -33,3 +33,12 @@ do
   echo -n "."
   sleep 2s
 done
+
+svn co svn://svn.code.sf.net/p/teca/TECA_assets@${TECA_ASSETS_REVISION} TECA_assets &
+svn_pid=$!
+
+while [ -n "$(ps -p $svn_pid -o pid=)" ]
+do
+  echo -n "."
+  sleep 2s
+done
diff --git a/test/travis_ci/install_ubuntu_14_04.sh b/test/travis_ci/install_ubuntu_14_04.sh
@@ -23,3 +23,4 @@ sudo tar -C /usr -x -z -f cmake-3.5.2-Linux-x86_64.tar.gz --strip-components=1
 
 # install data files.
 svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data
+svn co svn://svn.code.sf.net/p/teca/TECA_assets@${TECA_ASSETS_REVISION} TECA_assets
diff --git a/test/travis_ci/install_ubuntu_18_04.sh b/test/travis_ci/install_ubuntu_18_04.sh
@@ -32,3 +32,4 @@ pip3 install numpy mpi4py matplotlib torch
 
 # install data files.
 svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data
+svn co svn://svn.code.sf.net/p/teca/TECA_assets@${TECA_ASSETS_REVISION} TECA_assets