#include "cs_defs.h"
#include "bft_error.h"
Go to the source code of this file.
|
#define | CS_MALLOC(_ptr, _ni, _type) |
| Allocate memory for _ni elements of type _type. More...
|
|
#define | CS_MALLOC_HD(_ptr, _ni, _type, _mode) |
|
#define | CS_REALLOC(_ptr, _ni, _type) |
| Reallocate memory for _ni elements of type _type. More...
|
|
#define | CS_REALLOC_HD(_ptr, _ni, _type, _mode) |
|
#define | CS_FREE(_ptr) cs_mem_free(_ptr, #_ptr, __FILE__, __LINE__), _ptr = NULL |
|
#define | CS_MEMALIGN(_ptr, _align, _ni, _type) |
| Allocate aligned memory for _ni elements of type _type. More...
|
|
#define | cs_alloc_mode CS_ALLOC_HOST |
|
#define | cs_alloc_mode_read_mostly CS_ALLOC_HOST |
|
#define | cs_associate_device_ptr(_host_ptr, _ni, _size) ; |
| Associate device memory with a given host memory pointer. More...
|
|
#define | cs_disassociate_device_ptr(_host_ptr) ; |
| Detach device memory from a given host memory pointer. More...
|
|
#define | cs_set_alloc_mode(_host_ptr, mode) ; |
| Set allocation mode for an already allocated pointer. More...
|
|
#define | cs_set_alloc_mode_r(_host_ptr, mode) ; |
| Set allocation mode for an already allocated pointer using pass by reference semantics for the pointer (C++ only). More...
|
|
#define | cs_mem_advise_set_read_mostly(ptr) ; |
| Advise memory system that a given allocation will be mostly read. More...
|
|
#define | cs_mem_advise_unset_read_mostly(ptr) ; |
| Advise memory system that a given allocation will be mostly read. More...
|
|
|
int | cs_mem_stats (uint64_t *alloc_cur, uint64_t *alloc_max, uint64_t *n_allocs, uint64_t *n_reallocs, uint64_t *n_frees, uint64_t *n_current) |
| Return memory allocation stats, if available. More...
|
|
static void * | cs_get_device_ptr (void *ptr) |
| Return matching device pointer for a given pointer. More...
|
|
static const void * | cs_get_device_ptr_const (const void *ptr) |
| Return matching device pointer for a given constant pointer. More...
|
|
static const void * | cs_get_device_ptr_const_pf (const void *ptr) |
| Return matching device pointer for a given constant pointer, prefetching if applicable. More...
|
|
static cs_alloc_mode_t | cs_check_device_ptr (const void *ptr) |
| Check if a pointer is associated with a device. More...
|
|
static bool | cs_mem_is_device_ptr (const void *ptr) |
| Check if a pointer is a device (or shared) pointer. More...
|
|
static void | cs_sync_h2d (const void *ptr) |
| Synchronize data from host to device. More...
|
|
static void | cs_sync_d2h (void *ptr) |
| Synchronize data from device to host. More...
|
|
static void | cs_sync_d2h_if_needed (void *ptr) |
| Synchronize data from device to host, only if needed. More...
|
|
static void | cs_prefetch_h2d (void *ptr, size_t size) |
| Prefetch data from host to device. More...
|
|
static void | cs_prefetch_d2h (void *ptr, size_t size) |
| Prefetch data from device to host. More...
|
|
◆ cs_alloc_mode
◆ cs_alloc_mode_read_mostly
◆ cs_associate_device_ptr
#define cs_associate_device_ptr |
( |
|
_host_ptr, |
|
|
|
_ni, |
|
|
|
_size |
|
) |
| ; |
Associate device memory with a given host memory pointer.
If the host memory is already associated with the device, the existing device pointer is returned. Otherwise, a new device allocation is called and returned.
- Parameters
-
[in] | host_ptr | host pointer |
[in] | ni | number of elements |
[in] | size | element size |
- Returns
- pointer to allocated memory.
◆ cs_disassociate_device_ptr
#define cs_disassociate_device_ptr |
( |
|
_host_ptr | ) |
; |
Detach device memory from a given host memory pointer.
If the host memory is shared with the device (i.e. using CS_ALLOC_SHARED), device memory stays shared.
- Parameters
-
◆ CS_FREE
#define CS_FREE |
( |
|
_ptr | ) |
cs_mem_free(_ptr, #_ptr, __FILE__, __LINE__), _ptr = NULL |
◆ CS_MALLOC
CS_MALLOC |
( |
|
_ptr, |
|
|
|
_ni, |
|
|
|
_type |
|
) |
| |
Value:_ptr = (_type *) cs_mem_malloc(_ni, sizeof(_type), \
#_ptr, __FILE__, __LINE__)
Allocate memory for _ni elements of type _type.
This macro calls cs_mem_malloc(), automatically setting the allocated variable name and source file name and line arguments.
- Parameters
-
[out] | _ptr | pointer to allocated memory. |
[in] | _ni | number of elements. |
[in] | _type | element type. |
◆ CS_MALLOC_HD
#define CS_MALLOC_HD |
( |
|
_ptr, |
|
|
|
_ni, |
|
|
|
_type, |
|
|
|
_mode |
|
) |
| |
Value:_ptr = (_type *) cs_mem_malloc_hd(_mode, _ni, sizeof(_type), \
#_ptr, __FILE__, __LINE__)
◆ cs_mem_advise_set_read_mostly
#define cs_mem_advise_set_read_mostly |
( |
|
ptr | ) |
; |
Advise memory system that a given allocation will be mostly read.
- Parameters
-
[in] | ptr | pointer to allocation |
◆ cs_mem_advise_unset_read_mostly
#define cs_mem_advise_unset_read_mostly |
( |
|
ptr | ) |
; |
Advise memory system that a given allocation will be mostly read.
- Parameters
-
[in] | ptr | pointer to allocation |
◆ CS_MEMALIGN
CS_MEMALIGN |
( |
|
_ptr, |
|
|
|
_align, |
|
|
|
_ni, |
|
|
|
_type |
|
) |
| |
Value:_ptr = (_type *) cs_mem_memalign(_align, _ni, sizeof(_type), \
#_ptr, __FILE__, __LINE__)
Allocate aligned memory for _ni elements of type _type.
This macro calls cs_mem_memalign(), automatically setting the allocated variable name and source file name and line arguments.
- Parameters
-
[out] | _ptr | pointer to allocated memory. |
[in] | _align | alignment. |
[in] | _ni | number of elements. |
[in] | _type | element type. |
◆ CS_REALLOC
CS_REALLOC |
( |
|
_ptr, |
|
|
|
_ni, |
|
|
|
_type |
|
) |
| |
Value:_ptr = (_type *) cs_mem_realloc(_ptr, _ni, sizeof(_type), \
#_ptr, __FILE__, __LINE__)
Reallocate memory for _ni elements of type _type.
This macro calls cs_mem_realloc(), automatically setting the allocated variable name and source file name and line arguments.
- Parameters
-
[in,out] | _ptr | pointer to allocated memory. |
[in] | _ni | number of elements. |
[in] | _type | element type. |
◆ CS_REALLOC_HD
#define CS_REALLOC_HD |
( |
|
_ptr, |
|
|
|
_ni, |
|
|
|
_type, |
|
|
|
_mode |
|
) |
| |
Value:_ptr = (_type *) cs_mem_realloc_hd(_ptr, _mode, _ni, sizeof(_type), \
#_ptr, __FILE__, __LINE__)
◆ cs_set_alloc_mode
#define cs_set_alloc_mode |
( |
|
_host_ptr, |
|
|
|
mode |
|
) |
| ; |
Set allocation mode for an already allocated pointer.
If the allocation mode is different from the previous one, the associated memory will be reallocated with the desired mode, and the previous allocation freed.
- Parameters
-
[in,out] | host_ptr | pointer to host pointer to modify |
[in] | mode | desired allocation mode |
◆ cs_set_alloc_mode_r
#define cs_set_alloc_mode_r |
( |
|
_host_ptr, |
|
|
|
mode |
|
) |
| ; |
Set allocation mode for an already allocated pointer using pass by reference semantics for the pointer (C++ only).
In C++, this function is preferred to direct use of cs_set_alloc_mode, as it allows a more concise syntax and does not require additional casting.
- Parameters
-
[in,out] | host_ptr | reference to host pointer to modify |
[in] | mode | desired allocation mode |
◆ cs_alloc_mode_t
Allocation modes for accelerated code.
Enumerator |
---|
CS_ALLOC_HOST | allocation on host only
|
CS_ALLOC_HOST_DEVICE | allocation on host and device
|
CS_ALLOC_HOST_DEVICE_PINNED | allocation on host and device, using page-locked memory on host if possible
|
CS_ALLOC_HOST_DEVICE_SHARED | allocation on host and device, using mapped/shared memory
|
CS_ALLOC_DEVICE | allocation on device only
|
◆ cs_check_device_ptr()
Check if a pointer is associated with a device.
If separate pointers are used on the host and device, the host pointer should be used with this function.
- Returns
- allocation mode associated with pointer
◆ cs_get_device_ptr()
static void * cs_get_device_ptr |
( |
void * |
ptr | ) |
|
|
inlinestatic |
Return matching device pointer for a given pointer.
If separate pointers are used on the host and device, the host pointer should be used with this function.
If memory is not allocated on device yet at the call site, it will be allocated automatically by this function.
- Parameters
-
- Returns
- pointer to device memory.
◆ cs_get_device_ptr_const()
static const void * cs_get_device_ptr_const |
( |
const void * |
ptr | ) |
|
|
inlinestatic |
Return matching device pointer for a given constant pointer.
If separate pointers are used on the host and device, the host pointer should be used with this function.
If memory is not allocated on device yet at the call site, it will be allocated automatically by this function.
- Parameters
-
- Returns
- pointer to device memory.
◆ cs_get_device_ptr_const_pf()
static const void * cs_get_device_ptr_const_pf |
( |
const void * |
ptr | ) |
|
|
inlinestatic |
Return matching device pointer for a given constant pointer, prefetching if applicable.
If separate pointers are used on the host and device, the host pointer should be used with this function. In this case, it is assumed that the host and device values have already been synchronized, unless memory is not allocated on device yet at the call site, in which case it will be allocated automatically by this function.
- Parameters
-
- Returns
- pointer to device memory.
◆ cs_mem_is_device_ptr()
static bool cs_mem_is_device_ptr |
( |
const void * |
ptr | ) |
|
|
inlinestatic |
Check if a pointer is a device (or shared) pointer.
- Parameters
-
[in] | ptr | pointer to device data |
- Returns
- true if the pointer is usable from the device or null, false if available on host only or if query failed.
◆ cs_mem_stats()
int cs_mem_stats |
( |
uint64_t * |
alloc_cur, |
|
|
uint64_t * |
alloc_max, |
|
|
uint64_t * |
n_allocs, |
|
|
uint64_t * |
n_reallocs, |
|
|
uint64_t * |
n_frees, |
|
|
uint64_t * |
n_current |
|
) |
| |
Return memory allocation stats, if available.
Availability of statistics depends on the cs_mem_init options.
- Parameters
-
[out] | alloc_cur | current allocation size, or nullptr |
[out] | alloc_max | max allocation size, or nullptr |
[out] | n_allocs | total number of allocations, or nullptr |
[out] | n_reallocs | total number of reallocations, or nullptr |
[out] | n_frees | total number of frees, or nullptr |
[out] | n_current | total number of current allocations, or nullptr |
\return 1 if stats are available, O otherwise.
◆ cs_prefetch_d2h()
static void cs_prefetch_d2h |
( |
void * |
ptr, |
|
|
size_t |
size |
|
) |
| |
|
inlinestatic |
Prefetch data from device to host.
This function should only be used on arrays using shared host and device memory, shuch as those allocated using CS_ALLOC_HOST_DEVICE_SHARED. It should be usable on a subset of such an array.
- Parameters
-
[in,out] | ptr | pointer to data to prefetch |
[in] | size | number of bytes to prefetch |
◆ cs_prefetch_h2d()
static void cs_prefetch_h2d |
( |
void * |
ptr, |
|
|
size_t |
size |
|
) |
| |
|
inlinestatic |
Prefetch data from host to device.
This function should only be used on arrays using shared host and device memory, shuch as those allocated using CS_ALLOC_HOST_DEVICE_SHARED. It should be usable on a subset of such an array.
- Parameters
-
[in,out] | ptr | pointer to data to prefetch |
[in] | size | number of bytes to prefetch |
◆ cs_sync_d2h()
static void cs_sync_d2h |
( |
void * |
ptr | ) |
|
|
inlinestatic |
Synchronize data from device to host.
If separate allocations are used on the host and device (mode == CS_ALLOC_HOST_DEVICE), the host pointer should be passed to this function.
Depending on the allocaton type, this can imply a copy, data prefetch, or a no-op.
This function assumes the provided pointer was allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.
- Parameters
-
[in,out] | ptr | pointer to values to copy or prefetch |
◆ cs_sync_d2h_if_needed()
static void cs_sync_d2h_if_needed |
( |
void * |
ptr | ) |
|
|
inlinestatic |
Synchronize data from device to host, only if needed.
If separate allocations are used on the host and device (mode == CS_ALLOC_HOST_DEVICE), the host pointer should be passed to this function.
Depending on the allocation type, this can imply a copy, data prefetch, or a no-op.
No operation occurs if the provided pointer was not allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.
- Parameters
-
[in,out] | ptr | pointer to values to copy or prefetch |
◆ cs_sync_h2d()
static void cs_sync_h2d |
( |
const void * |
ptr | ) |
|
|
inlinestatic |
Synchronize data from host to device.
If separate pointers are used on the host and device, the host pointer should be used with this function.
Depending on the allocation type, this can imply a copy, data prefetch, or a no-op.
This function assumes the provided pointer was allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.
- Parameters
-
[in,out] | ptr | host pointer to values to copy or prefetch |