Skip to content

Commit

Permalink
copiers use PM memcpy equivalent when possible
Browse files Browse the repository at this point in the history 
when possible use the progrmamming model equivalent of memcpy. This had
inadvertantly been disabled. This avoids a kernel launch.
  • Loading branch information
@burlen
burlen committed Sep 25, 2023
1 parent 144928f commit 1d6bc13
Show file tree
Hide file tree
Showing 10 changed files with 264 additions and 341 deletions.
34 changes: 34 additions & 0 deletions hamr_copier_traits.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#include "hamr_config.h"
#include <type_traits>

namespace hamr
{
/// @name type trait that enables object copy
///@{
template <typename T, typename U, bool val = (!std::is_arithmetic<T>::value || !std::is_arithmetic<U>::value)> struct use_object_copier : std::false_type {};
template <typename T, typename U> struct use_object_copier<T, U, true> : std::true_type {};
template <typename T, typename U> using use_object_copier_t = typename std::enable_if<use_object_copier<T,U>::value>::type;
///@}


/// @name type trait that enables POD copy from different types
///@{
#if defined(HAMR_ENABLE_OBJECTS)
template <typename T, typename U, bool val = (!std::is_same<T,U>::value)> struct use_cons_copier : std::false_type {};
template <typename T, typename U> struct use_cons_copier<T, U, true> : std::true_type {};
template <typename T, typename U> using use_cons_copier_t = typename std::enable_if<use_cons_copier<T,U>::value>::type;
#else
template <typename T, typename U, bool val = (!std::is_same<T,U>::value && std::is_arithmetic<T>::value)> struct use_cons_copier : std::false_type {};
template <typename T, typename U> struct use_cons_copier<T, U, true> : std::true_type {};
template <typename T, typename U> using use_cons_copier_t = typename std::enable_if<use_cons_copier<T,U>::value>::type;
#endif
///@}

/// @name type trait that enables POD copy from the same types
///@{
template <typename T, typename U, bool obj = (std::is_same<T,U>::value && std::is_arithmetic<T>::value)> struct use_bytes_copier : std::false_type {};
template <typename T, typename U> struct use_bytes_copier<T, U, true> : std::true_type {};
template <typename T, typename U> using use_bytes_copier_t = typename std::enable_if<use_bytes_copier<T,U>::value>::type;
///@}

}
29 changes: 9 additions & 20 deletions hamr_cuda_copy_async.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,18 @@
#include "hamr_cuda_copy_async.h"
#include "hamr_cuda_copy_async_impl.h"

#if !defined(HAMR_ENABLE_OBJECTS)

#define hamr_cuda_copy_async_instantiate_(T, U) \
template int hamr::copy_to_cuda_from_host<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem, void *); \
template int hamr::copy_to_cuda_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem, void *); \
template int hamr::copy_to_cuda_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, int src_device, size_t n_elem, void *); \
template int hamr::copy_to_host_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem, void *);

#else

#define hamr_cuda_copy_async_instantiate_(T, U) \
template int hamr::copy_to_cuda_from_host<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem); \
template int hamr::copy_to_cuda_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem); \
template int hamr::copy_to_cuda_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, int src_device, size_t n_elem); \
template int hamr::copy_to_host_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem);

#endif
template int hamr::copy_to_cuda_from_host<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem, hamr::use_cons_copier_t<T,U> *); \
template int hamr::copy_to_cuda_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem, hamr::use_cons_copier_t<T,U> *); \
template int hamr::copy_to_cuda_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, int src_device, size_t n_elem, hamr::use_cons_copier_t<T,U> *); \
template int hamr::copy_to_host_from_cuda<T,U>(cudaStream_t strm, T *dest, const U *src, size_t n_elem, hamr::use_cons_copier_t<T,U> *);

#define hamr_cuda_copy_async_instantiate__(T) \
template int hamr::copy_to_cuda_from_host<T>(cudaStream_t strm, T *dest, const T *src, size_t n_elem, void *); \
template int hamr::copy_to_cuda_from_cuda<T>(cudaStream_t strm, T *dest, const T *src, size_t n_elem, void *); \
template int hamr::copy_to_cuda_from_cuda<T>(cudaStream_t strm, T *dest, const T *src, int src_device, size_t n_elem, void *); \
template int hamr::copy_to_host_from_cuda<T>(cudaStream_t strm, T *dest, const T *src, size_t n_elem, void *);
template int hamr::copy_to_cuda_from_host<T,T>(cudaStream_t strm, T *dest, const T *src, size_t n_elem, hamr::use_bytes_copier_t<T,T> *); \
template int hamr::copy_to_cuda_from_cuda<T,T>(cudaStream_t strm, T *dest, const T *src, size_t n_elem, hamr::use_bytes_copier_t<T,T> *); \
template int hamr::copy_to_cuda_from_cuda<T,T>(cudaStream_t strm, T *dest, const T *src, int src_device, size_t n_elem, hamr::use_bytes_copier_t<T,T> *); \
template int hamr::copy_to_host_from_cuda<T,T>(cudaStream_t strm, T *dest, const T *src, size_t n_elem, hamr::use_bytes_copier_t<T,T> *); \


hamr_cuda_copy_async_instantiate__(float)
hamr_cuda_copy_async_instantiate__(double)
Expand Down
89 changes: 38 additions & 51 deletions hamr_cuda_copy_async.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
#define hamr_cuda_copy_async_h

#include "hamr_config.h"
#include "hamr_copier_traits.h"

#include <memory>
#include <type_traits>

Expand All @@ -20,8 +22,9 @@ namespace hamr
*/
template <typename T, typename U>
int copy_to_cuda_from_host(cudaStream_t str, T *dest, const U *src, size_t n_elem,
typename std::enable_if<!std::is_arithmetic<T>::value>::type * = nullptr);
#else
hamr::use_object_copier_t<T,U> * = nullptr);
#endif

/** Copies an array to the active CUDA device (fast path for arrays of
* arithmetic types of the same type).
*
Expand All @@ -32,10 +35,9 @@ int copy_to_cuda_from_host(cudaStream_t str, T *dest, const U *src, size_t n_ele
*
* @returns 0 if there were no errors
*/
template <typename T>
int copy_to_cuda_from_host(cudaStream_t str, T *dest, const T *src, size_t n_elem,
typename std::enable_if<std::is_arithmetic<T>::value>::type * = nullptr);
#endif
template <typename T, typename U>
int copy_to_cuda_from_host(cudaStream_t str, T *dest, const U *src, size_t n_elem,
hamr::use_bytes_copier_t<T,U> * = nullptr);

/** Copies an array to the active CUDA device.
*
Expand All @@ -47,11 +49,9 @@ int copy_to_cuda_from_host(cudaStream_t str, T *dest, const T *src, size_t n_ele
* @returns 0 if there were no errors
*/
template <typename T, typename U>
int copy_to_cuda_from_host(cudaStream_t str, T *dest, const U *src, size_t n_elem
#if !defined(HAMR_ENABLE_OBJECTS)
,typename std::enable_if<std::is_arithmetic<T>::value>::type * = nullptr
#endif
);
int copy_to_cuda_from_host(cudaStream_t str, T *dest, const U *src, size_t n_elem,
hamr::use_cons_copier_t<T,U> * = nullptr);


#if !defined(HAMR_ENABLE_OBJECTS)
/** Copies an array on the active CUDA device.
Expand All @@ -65,8 +65,9 @@ int copy_to_cuda_from_host(cudaStream_t str, T *dest, const U *src, size_t n_ele
*/
template <typename T, typename U>
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_elem,
typename std::enable_if<!std::is_arithmetic<T>::value>::type * = nullptr);
#else
hamr::use_object_copier_t<T,U> * = nullptr);
#endif

/** Ccopies an array on the active CUAD device (fast path for arrays of
* arithmetic types of the same type).
*
Expand All @@ -77,10 +78,10 @@ int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_ele
*
* @returns 0 if there were no errors
*/
template <typename T>
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const T *src, size_t n_elem,
typename std::enable_if<std::is_arithmetic<T>::value>::type * = nullptr);
#endif
template <typename T, typename U>
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_elem,
hamr::use_bytes_copier_t<T,U> * = nullptr);


/** Copies an array on the active CUDA device.
*
Expand All @@ -92,11 +93,8 @@ int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const T *src, size_t n_ele
* @returns 0 if there were no errors
*/
template <typename T, typename U>
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_elem
#if !defined(HAMR_ENABLE_OBJECTS)
,typename std::enable_if<std::is_arithmetic<T>::value>::type * = nullptr
#endif
);
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_elem,
hamr::use_cons_copier_t<T,U> * = nullptr);

#if !defined(HAMR_ENABLE_OBJECTS)
/** Copies an array to the active CUDA device from the named CUDA device,
Expand All @@ -110,10 +108,10 @@ int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_ele
* @returns 0 if there were no errors
*/
template <typename T, typename U>
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest,
const U *src, int src_device, size_t n_elem,
typename std::enable_if<!std::is_arithmetic<T>::value>::type * = nullptr);
#else
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src,
int src_device, size_t n_elem, hamr::use_object_copier_t<T,U> * = nullptr);
#endif

/** Copies an array to the active CUDA device from the named CUDA device, (fast
* path for arrays of arithmetic types of the same type).
*
Expand All @@ -125,11 +123,9 @@ int copy_to_cuda_from_cuda(cudaStream_t str, T *dest,
*
* @returns 0 if there were no errors
*/
template <typename T>
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest,
const T *src, int src_device, size_t n_elem,
typename std::enable_if<std::is_arithmetic<T>::value>::type * = nullptr);
#endif
template <typename T, typename U>
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src,
int src_device, size_t n_elem, hamr::use_bytes_copier_t<T,U> * = nullptr);

/** Copies an array on the active CUDA device.
*
Expand All @@ -142,12 +138,8 @@ int copy_to_cuda_from_cuda(cudaStream_t str, T *dest,
* @returns 0 if there were no errors
*/
template <typename T, typename U>
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest,
const U *src, int src_device, size_t n_elem
#if !defined(HAMR_ENABLE_OBJECTS)
,typename std::enable_if<std::is_arithmetic<T>::value>::type * = nullptr
#endif
);
int copy_to_cuda_from_cuda(cudaStream_t str, T *dest, const U *src,
int src_device, size_t n_elem, hamr::use_cons_copier_t<T,U> * = nullptr);

#if !defined(HAMR_ENABLE_OBJECTS)
/** Copies an array from the active CUDA device.
Expand All @@ -160,10 +152,10 @@ int copy_to_cuda_from_cuda(cudaStream_t str, T *dest,
* @returns 0 if there were no errors
*/
template <typename T, typename U>
int copy_to_host_from_cuda(cudaStream_t str, T *dest,
const U *src, size_t n_elem,
typename std::enable_if<!std::is_arithmetic<T>::value>::type * = nullptr);
#else
int copy_to_host_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_elem,
hamr::use_object_copier_t<T,U> * = nullptr);
#endif

/** Copies an array from the active CUDA device (fast path for arrays of
* arithmetic types of the same type).
*
Expand All @@ -174,11 +166,9 @@ int copy_to_host_from_cuda(cudaStream_t str, T *dest,
*
* @returns 0 if there were no errors
*/
template <typename T>
int copy_to_host_from_cuda(cudaStream_t str, T *dest,
const T *src, size_t n_elem,
typename std::enable_if<std::is_arithmetic<T>::value>::type * = nullptr);
#endif
template <typename T, typename U>
int copy_to_host_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_elem,
hamr::use_bytes_copier_t<T,U> * = nullptr);

/** Copies an array from the active CUDA device.
*
Expand All @@ -188,11 +178,8 @@ int copy_to_host_from_cuda(cudaStream_t str, T *dest,
* @returns 0 if there were no errors
*/
template <typename T, typename U>
int copy_to_host_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_elem
#if !defined(HAMR_ENABLE_OBJECTS)
,typename std::enable_if<std::is_arithmetic<T>::value>::type * = nullptr
#endif
);
int copy_to_host_from_cuda(cudaStream_t str, T *dest, const U *src, size_t n_elem,
hamr::use_cons_copier_t<T,U> * = nullptr);

}

Expand Down
Loading

0 comments on commit 1d6bc13

Please sign in to comment.