Skip to content

Commit

Permalink
[HIP] Refactor HIP pointer type detection
Browse files Browse the repository at this point in the history
  • Loading branch information
@GeorgeWeb
GeorgeWeb committed Apr 15, 2024
1 parent 15233fd commit 776dfef
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 67 deletions.
17 changes: 17 additions & 0 deletions source/adapters/hip/common.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,23 @@
#include <hip/hip_runtime.h>
#include <ur/ur.hpp>

static inline unsigned int
getHipMemoryType(const hipPointerAttribute_t &Attributes) {
#if HIP_VERSION >= 50600000
return Attributes.type;
#else
return Attributes.memoryType;
#endif
}

// Error code hipErrorInvalidValue returned from hipPointerGetAttributes
// for a non-null pointer refers to an OS-allocation, hence we can work
// with the assumption that this is a pointer to a pageable host memory.
// NOTE Since ROCm version 6.0.0, the enum hipMemoryType can also be marked
// hipMemoryTypeUnregistered explicitly but until then that does not exist.
ur::Result<unsigned int>
getMemoryTypePointerAttributeOrInvalidValue(const void *Ptr);

// Before ROCm 6, hipify doesn't support cuArrayGetDescriptor, on AMD the
// hipArray can just be indexed, but on NVidia it is an opaque type and needs to
// go through cuArrayGetDescriptor so implement a utility function to get the
Expand Down
115 changes: 60 additions & 55 deletions source/adapters/hip/enqueue.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,18 @@ void guessLocalWorkSize(ur_device_handle_t Device, size_t *ThreadsPerBlock,
MaxBlockDim, MaxThreadsPerBlock[0]);
}

ur::Result<unsigned int>
getMemoryTypePointerAttributeOrInvalidValue(const void *Ptr) {
hipPointerAttribute_t Attributes{};
hipError_t Ret = hipPointerGetAttributes(&Attributes, Ptr);
if (Ret == hipErrorInvalidValue && Ptr)
return mapErrorUR(Ret);
// Direct usage of the function, instead of UR_CHECK_ERROR, to get line
// offset.
checkErrorUR(Ret, __func__, __LINE__ - 4, __FILE__);
return getHipMemoryType(Attributes);
}

namespace {

ur_result_t setHipMemAdvise(const void *DevPtr, const size_t Size,
Expand Down Expand Up @@ -1614,6 +1626,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMFill2D(
return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
}

// Determine the memory type for shared allocations.
//
// ROCm 5.7.1 finally started updating the type attribute member to
// hipMemoryTypeManaged for shared memory allocations(hipMallocManaged).
// Hence, we use a separate query that verifies the pointer use via flags.
static inline void
getManagedPointerTypeLocation([[maybe_unused]] unsigned int &memType,
[[maybe_unused]] void *pMem) {
#if HIP_VERSION_MAJOR >= 5
UR_CHECK_ERROR(
hipPointerGetAttribute(&memType, HIP_POINTER_ATTRIBUTE_MEMORY_TYPE,
reinterpret_cast<hipDeviceptr_t>(pMem)));
// Verify the memory location; can only be device or host.
UR_ASSERT(memType == hipMemoryTypeHost || memType == hipMemoryTypeDevice,
UR_RESULT_ERROR_INVALID_MEM_OBJECT);
#endif
}

/// 2D Memcpy API
///
/// \param hQueue is the queue to submit to
Expand Down Expand Up @@ -1654,65 +1684,40 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueUSMMemcpy2D(
// the copies since ROCm 5.6.0+. See: https://github.com/ROCm/clr/issues/40
// TODO: Add maximum HIP_VERSION when bug has been fixed.
#if HIP_VERSION >= 50600000
hipPointerAttribute_t srcAttribs{};
hipPointerAttribute_t dstAttribs{};

// Determine if pSrc and/or pDst are system allocated pageable host memory.
bool srcIsSystemAlloc{false};
bool dstIsSystemAlloc{false};

hipError_t hipRes{};
// Error code hipErrorInvalidValue returned from hipPointerGetAttributes
// for a non-null pointer refers to an OS-allocation, hence we can work
// with the assumption that this is a pointer to a pageable host memory.
// Since ROCm version 6.0.0, the enum hipMemoryType can also be marked as
// hipMemoryTypeUnregistered explicitly to relay that information better.
// This means we cannot rely on any attribute result, hence we just mark
// the pointer handle as system allocated pageable host memory.
// The HIP runtime can handle the registering/unregistering of the memory
// as long as the right copy-kind (direction) is provided to hipMemcpy2D*.
hipRes = hipPointerGetAttributes(&srcAttribs, pSrc);
if (hipRes == hipErrorInvalidValue && pSrc)
srcIsSystemAlloc = true;
hipRes = hipPointerGetAttributes(&dstAttribs, (const void *)pDst);
if (hipRes == hipErrorInvalidValue && pDst)
dstIsSystemAlloc = true;
bool srcIsHost{false};
bool srcIsDevice{false};
bool dstIsHost{false};
bool dstIsDevice{false};

static const auto isHostMemoryType = [](unsigned int Type) -> bool {
#if HIP_VERSION_MAJOR >= 6
srcIsSystemAlloc |= srcAttribs.type == hipMemoryTypeUnregistered;
dstIsSystemAlloc |= dstAttribs.type == hipMemoryTypeUnregistered;
return Type == hipMemoryTypeHost || Type == hipMemoryTypeUnregistered;
#else
return Type == hipMemoryTypeHost;
#endif
};

unsigned int srcMemType{srcAttribs.type};
unsigned int dstMemType{dstAttribs.type};

// ROCm 5.7.1 finally started updating the type attribute member to
// hipMemoryTypeManaged for shared memory allocations(hipMallocManaged).
// Hence, we use a separate query that verifies the pointer use via flags.
#if HIP_VERSION >= 50700001
// Determine the source/destination memory type for shared allocations.
//
// NOTE: The hipPointerGetAttribute API is marked as [BETA] and fails with
// exit code -11 when passing a system allocated pointer to it.
if (!srcIsSystemAlloc && srcAttribs.isManaged) {
UR_ASSERT(srcAttribs.hostPointer && srcAttribs.devicePointer,
UR_RESULT_ERROR_INVALID_VALUE);
UR_CHECK_ERROR(hipPointerGetAttribute(
&srcMemType, HIP_POINTER_ATTRIBUTE_MEMORY_TYPE,
reinterpret_cast<hipDeviceptr_t>(const_cast<void *>(pSrc))));
}
if (!dstIsSystemAlloc && dstAttribs.isManaged) {
UR_ASSERT(dstAttribs.hostPointer && dstAttribs.devicePointer,
UR_RESULT_ERROR_INVALID_VALUE);
UR_CHECK_ERROR(
hipPointerGetAttribute(&dstMemType, HIP_POINTER_ATTRIBUTE_MEMORY_TYPE,
reinterpret_cast<hipDeviceptr_t>(pDst)));
auto srcTypeRes = getMemoryTypePointerAttributeOrInvalidValue(pSrc);
if (srcTypeRes.get_error() == UR_RESULT_ERROR_INVALID_VALUE)
srcIsHost = true;
if (srcTypeRes) {
unsigned int srcMemType = *srcTypeRes.get_value();
if (srcMemType == hipMemoryTypeManaged)
getManagedPointerTypeLocation(srcMemType, const_cast<void *>(pSrc));
srcIsHost = IsHostMemoryType(srcMemType);
srcIsDevice = srcMemType == hipMemoryTypeDevice;
}

auto dstTypeRes = getMemoryTypePointerAttributeOrInvalidValue((const void *)pDst);
if (dstTypeRes.get_error() == UR_RESULT_ERROR_INVALID_VALUE)
dstIsHost = true;
if (dstTypeRes) {
unsigned int dstMemType = *dstTypeRes.get_value();
if (dstMemType == hipMemoryTypeManaged)
getManagedPointerTypeLocation(dstMemType, pDst);
dstIsHost = IsHostMemoryType(dstMemType);
dstIsDevice = dstMemType == hipMemoryTypeDevice;
}
#endif

const bool srcIsHost{(srcMemType == hipMemoryTypeHost) || srcIsSystemAlloc};
const bool srcIsDevice{srcMemType == hipMemoryTypeDevice};
const bool dstIsHost{(dstMemType == hipMemoryTypeHost) || dstIsSystemAlloc};
const bool dstIsDevice{dstMemType == hipMemoryTypeDevice};

unsigned int cpyKind{};
if (srcIsHost && dstIsHost)
Expand Down
15 changes: 3 additions & 12 deletions source/adapters/hip/usm.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,7 @@ USMFreeImpl([[maybe_unused]] ur_context_handle_t hContext, void *pMem) {
try {
hipPointerAttribute_t hipPointerAttributeType;
UR_CHECK_ERROR(hipPointerGetAttributes(&hipPointerAttributeType, pMem));
#if HIP_VERSION >= 50600000
const auto Type = hipPointerAttributeType.type;
#else
const auto Type = hipPointerAttributeType.memoryType;
#endif
const unsigned int Type = getHipMemoryType(hipPointerAttributeType);
UR_ASSERT(Type == hipMemoryTypeDevice || Type == hipMemoryTypeHost ||
Type == hipMemoryTypeManaged,
UR_RESULT_ERROR_INVALID_MEM_OBJECT);
Expand Down Expand Up @@ -169,19 +165,14 @@ urUSMGetMemAllocInfo(ur_context_handle_t hContext, const void *pMem,
// Direct usage of the function, instead of UR_CHECK_ERROR, so we can get
// the line offset.
checkErrorUR(Ret, __func__, __LINE__ - 5, __FILE__);
const unsigned int Value = getHipMemoryType(hipPointerAttributeType);
// ROCm 6.0.0 introduces hipMemoryTypeUnregistered in the hipMemoryType
// enum to mark unregistered allocations (i.e., via system allocators).
#if HIP_VERSION_MAJOR >= 6
if (hipPointerAttributeType.type == hipMemoryTypeUnregistered) {
if (Value == hipMemoryTypeUnregistered) {
// pointer not known to the HIP subsystem
return ReturnValue(UR_USM_TYPE_UNKNOWN);
}
#endif
unsigned int Value;
#if HIP_VERSION >= 50600000
Value = hipPointerAttributeType.type;
#else
Value = hipPointerAttributeType.memoryType;
#endif
UR_ASSERT(Value == hipMemoryTypeDevice || Value == hipMemoryTypeHost ||
Value == hipMemoryTypeManaged,
Expand Down

0 comments on commit 776dfef

Please sign in to comment.