cufile.h(3)

NAME
cufile.h - cuFile C APIs

SYNOPSIS

  Data Structures

struct CUfileDescr_t
struct CUfileDrvProps
struct CUfileError
cufileop status string
struct CUfileFSOps
struct CUfileIOEvents
struct CUfileIOParams
struct CUfileOpCounter
Counter structure for tracking operation successes and failures.
struct CUfilePerGpuStats
Per-GPU statistics structure used in Level 3.
struct cufileRDMAInfo
struct CUfileStatsLevel1
Level 1 Statistics: Basic IO and operation counters.
struct CUfileStatsLevel2
Level 2 Statistics: Includes Level 1 plus size histograms and detailed metrics.
struct CUfileStatsLevel3
Level 3 Statistics: Includes Level 2 plus per-GPU and subsystem statistics.

  Macros

#define CU_FILE_CUDA_ERR(status) ((status).cu_err)
#define CU_FILE_RDMA_REGISTER 1
#define CU_FILE_RDMA_RELAXED_ORDERING (1<<1)
#define CU_FILE_STREAM_FIXED_BUF_OFFSET 1
#define CU_FILE_STREAM_FIXED_FILE_OFFSET 2
#define CU_FILE_STREAM_FIXED_FILE_SIZE 4
#define CU_FILE_STREAM_PAGE_ALIGNED_INPUTS 8
#define CUFILE_ERRSTR(err) cufileop_status_error((CUfileOpError)abs((err)))
#define CUFILE_GPU_UUID_LEN 16
#define cuFileDriverClose cuFileDriverClose_v2
#define IS_CUDA_ERR(status) ((status).err == CU_FILE_CUDA_DRIVER_ERROR)
#define IS_CUFILE_ERR(err) (abs((err)) > CUFILEOP_BASE_ERR)
error macros to inspect error status of type CUfileOpError

  Typedefs

typedef enum CUFileArrayConfigParameter_t CUFileArrayConfigParameter_t
typedef void * CUfileBatchHandle_t
typedef enum cufileBatchMode CUfileBatchMode_t
typedef enum CUFileBoolConfigParameter_t CUFileBoolConfigParameter_t
typedef struct CUfileDescr_t CUfileDescr_t
typedef enum CUfileDriverControlFlags CUfileDriverControlFlags_t
typedef enum CUfileDriverStatusFlags CUfileDriverStatusFlags_t
typedef struct CUfileDrvProps CUfileDrvProps_t
typedef struct CUfileError CUfileError_t
cufileop status string
typedef enum CUfileFeatureFlags CUfileFeatureFlags_t
typedef struct CUfileFSOps CUfileFSOps_t
typedef void * CUfileHandle_t
File handle type.
typedef struct CUfileIOEvents CUfileIOEvents_t
typedef struct CUfileIOParams CUfileIOParams_t
typedef enum CUfileOpcode CUfileOpcode_t
typedef struct CUfileOpCounter CUfileOpCounter_t
Counter structure for tracking operation successes and failures.
typedef struct CUfilePerGpuStats CUfilePerGpuStats_t
Per-GPU statistics structure used in Level 3.
typedef struct cufileRDMAInfo cufileRDMAInfo_t
typedef enum CUFileSizeTConfigParameter_t CUFileSizeTConfigParameter_t
typedef struct CUfileStatsLevel1 CUfileStatsLevel1_t
Level 1 Statistics: Basic IO and operation counters.
typedef struct CUfileStatsLevel2 CUfileStatsLevel2_t
Level 2 Statistics: Includes Level 1 plus size histograms and detailed metrics.
typedef struct CUfileStatsLevel3 CUfileStatsLevel3_t
Level 3 Statistics: Includes Level 2 plus per-GPU and subsystem statistics.
typedef enum CUFILEStatus_enum CUfileStatus_t
typedef enum CUFileStringConfigParameter_t CUFileStringConfigParameter_t
typedef struct sockaddr sockaddr_t

  Enumerations

enum CUFileArrayConfigParameter_t { CUFILE_PARAM_POSIX_POOL_SLAB_SIZE_KB, CUFILE_PARAM_POSIX_POOL_SLAB_COUNT }
enum cufileBatchMode { CUFILE_BATCH = 1 }
enum CUFileBoolConfigParameter_t { CUFILE_PARAM_PROPERTIES_USE_POLL_MODE, CUFILE_PARAM_PROPERTIES_ALLOW_COMPAT_MODE, CUFILE_PARAM_FORCE_COMPAT_MODE, CUFILE_PARAM_FS_MISC_API_CHECK_AGGRESSIVE, CUFILE_PARAM_EXECUTION_PARALLEL_IO, CUFILE_PARAM_PROFILE_NVTX, CUFILE_PARAM_PROPERTIES_ALLOW_SYSTEM_MEMORY, CUFILE_PARAM_USE_PCIP2PDMA, CUFILE_PARAM_PREFER_IO_URING, CUFILE_PARAM_FORCE_ODIRECT_MODE, CUFILE_PARAM_SKIP_TOPOLOGY_DETECTION, CUFILE_PARAM_STREAM_MEMOPS_BYPASS }
enum CUfileDriverControlFlags { CU_FILE_USE_POLL_MODE = 0, CU_FILE_ALLOW_COMPAT_MODE = 1 }
enum CUfileDriverStatusFlags { CU_FILE_LUSTRE_SUPPORTED = 0, CU_FILE_WEKAFS_SUPPORTED = 1, CU_FILE_NFS_SUPPORTED = 2, CU_FILE_GPFS_SUPPORTED = 3, CU_FILE_NVME_SUPPORTED = 4, CU_FILE_NVMEOF_SUPPORTED = 5, CU_FILE_SCSI_SUPPORTED = 6, CU_FILE_SCALEFLUX_CSD_SUPPORTED = 7, CU_FILE_NVMESH_SUPPORTED = 8, CU_FILE_BEEGFS_SUPPORTED = 9, CU_FILE_NVME_P2P_SUPPORTED = 11, CU_FILE_SCATEFS_SUPPORTED = 12 }
enum CUfileFeatureFlags { CU_FILE_DYN_ROUTING_SUPPORTED = 0, CU_FILE_BATCH_IO_SUPPORTED = 1, CU_FILE_STREAMS_SUPPORTED = 2, CU_FILE_PARALLEL_IO_SUPPORTED = 3 }
enum CUfileFileHandleType { CU_FILE_HANDLE_TYPE_OPAQUE_FD = 1, CU_FILE_HANDLE_TYPE_OPAQUE_WIN32 = 2, CU_FILE_HANDLE_TYPE_USERSPACE_FS = 3 }
enum CUfileOpcode { CUFILE_READ = 0, CUFILE_WRITE }
enum CUFileSizeTConfigParameter_t { CUFILE_PARAM_PROFILE_STATS, CUFILE_PARAM_EXECUTION_MAX_IO_QUEUE_DEPTH, CUFILE_PARAM_EXECUTION_MAX_IO_THREADS, CUFILE_PARAM_EXECUTION_MIN_IO_THRESHOLD_SIZE_KB, CUFILE_PARAM_EXECUTION_MAX_REQUEST_PARALLELISM, CUFILE_PARAM_PROPERTIES_MAX_DIRECT_IO_SIZE_KB, CUFILE_PARAM_PROPERTIES_MAX_DEVICE_CACHE_SIZE_KB, CUFILE_PARAM_PROPERTIES_PER_BUFFER_CACHE_SIZE_KB, CUFILE_PARAM_PROPERTIES_MAX_DEVICE_PINNED_MEM_SIZE_KB, CUFILE_PARAM_PROPERTIES_IO_BATCHSIZE, CUFILE_PARAM_POLLTHRESHOLD_SIZE_KB, CUFILE_PARAM_PROPERTIES_BATCH_IO_TIMEOUT_MS }
enum CUFILEStatus_enum { CUFILE_WAITING = 0x000001, CUFILE_PENDING = 0x000002, CUFILE_INVALID = 0x000004, CUFILE_CANCELED = 0x000008, CUFILE_COMPLETE = 0x0000010, CUFILE_TIMEOUT = 0x0000020, CUFILE_FAILED = 0x0000040 }
enum CUFileStringConfigParameter_t { CUFILE_PARAM_LOGGING_LEVEL, CUFILE_PARAM_ENV_LOGFILE_PATH, CUFILE_PARAM_LOG_DIR }

  Functions

CUfileError_t cuFileBatchIOCancel (CUfileBatchHandle_t batch_idp)
void cuFileBatchIODestroy (CUfileBatchHandle_t batch_idp)
CUfileError_t cuFileBatchIOGetStatus (CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned *nr, CUfileIOEvents_t *iocbp, struct timespec *timeout)
CUfileError_t cuFileBatchIOSetUp (CUfileBatchHandle_t *batch_idp, unsigned nr)
CUfileError_t cuFileBatchIOSubmit (CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t *iocbp, unsigned int flags)
CUfileError_t cuFileBufDeregister (const void *bufPtr_base)
deregister an already registered device or host memory from cuFile
CUfileError_t cuFileBufRegister (const void *bufPtr_base, size_t length, int flags)
register an existing cudaMalloced memory with cuFile to pin for GPUDirect Storage access or register host allocated memory with cuFile.
CUfileError_t cuFileDriverClose_v2 (void)
reset the cuFile library and release the nvidia-fs driver
CUfileError_t cuFileDriverGetProperties (CUfileDrvProps_t *props)
Gets the Driver session properties.
CUfileError_t cuFileDriverOpen (void)
Initialize the cuFile library and open the nvidia-fs driver.
CUfileError_t cuFileDriverSetMaxCacheSize (size_t max_cache_size)
Control parameter to set maximum GPU memory reserved per device by the library for internal buffering.
CUfileError_t cuFileDriverSetMaxDirectIOSize (size_t max_direct_io_size)
Control parameter to set max IO size(KB) used by the library to talk to nvidia-fs driver.
CUfileError_t cuFileDriverSetMaxPinnedMemSize (size_t max_pinned_size)
Sets maximum buffer space that is pinned in KB for use by cuFileBufRegister.
CUfileError_t cuFileDriverSetPollMode (bool poll, size_t poll_threshold_size)
Sets whether the Read/Write APIs use polling to do IO operations.
CUfileError_t cuFileGetBARSizeInKB (int gpuIndex, size_t *barSize)
CUfileError_t cuFileGetParameterBool (CUFileBoolConfigParameter_t param, bool *value)
CUfileError_t cuFileGetParameterMinMaxValue (CUFileSizeTConfigParameter_t param, size_t *min_value, size_t *max_value)
Get both the minimum and maximum settable values for a given size_t parameter in a single call.
CUfileError_t cuFileGetParameterPosixPoolSlabArray (size_t *size_values, size_t *count_values, int len)
Get both POSIX pool slab size and count parameters as a pair.
CUfileError_t cuFileGetParameterSizeT (CUFileSizeTConfigParameter_t param, size_t *value)
CUfileError_t cuFileGetParameterString (CUFileStringConfigParameter_t param, char *desc_str, int len)
CUfileError_t cuFileGetStatsL1 (CUfileStatsLevel1_t *stats)
Get Level 1 cuFile statistics.
CUfileError_t cuFileGetStatsL2 (CUfileStatsLevel2_t *stats)
Get Level 2 cuFile statistics.
CUfileError_t cuFileGetStatsL3 (CUfileStatsLevel3_t *stats)
Get Level 3 cuFile statistics.
CUfileError_t cuFileGetStatsLevel (int *level)
Get the current level of statistics collection for cuFile operations.
CUfileError_t cuFileGetVersion (int *version)
void cuFileHandleDeregister (CUfileHandle_t fh)
releases a registered filehandle from cuFile
CUfileError_t cuFileHandleRegister (CUfileHandle_t *fh, CUfileDescr_t *descr)
cuFileHandleRegister is required, and performs extra checking that is memoized to provide increased performance on later cuFile operations.
ssize_t cuFileRead (CUfileHandle_t fh, void *bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset)
read data from a registered file handle to a specified device or host memory
CUfileError_t cuFileReadAsync (CUfileHandle_t fh, void *bufPtr_base, size_t *size_p, off_t *file_offset_p, off_t *bufPtr_offset_p, ssize_t *bytes_read_p, CUstream stream)
CUfileError_t cuFileSetParameterBool (CUFileBoolConfigParameter_t param, bool value)
CUfileError_t cuFileSetParameterPosixPoolSlabArray (const size_t *size_values, const size_t *count_values, int len)
Set both POSIX pool slab size and count parameters as a pair.
CUfileError_t cuFileSetParameterSizeT (CUFileSizeTConfigParameter_t param, size_t value)
CUfileError_t cuFileSetParameterString (CUFileStringConfigParameter_t param, const char *desc_str)
CUfileError_t cuFileSetStatsLevel (int level)
Set the level of statistics collection for cuFile operations. This will override the cufile.json settings for stats.
CUfileError_t cuFileStatsReset (void)
Reset all cuFile statistics counters.
CUfileError_t cuFileStatsStart (void)
Start collecting cuFile statistics.
CUfileError_t cuFileStatsStop (void)
Stop collecting cuFile statistics.
CUfileError_t cuFileStreamDeregister (CUstream stream)
CUfileError_t cuFileStreamRegister (CUstream stream, unsigned flags)
long cuFileUseCount (void)
returns use count of cufile drivers at that moment by the process.
ssize_t cuFileWrite (CUfileHandle_t fh, const void *bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset)
write data from a specified device or host memory to a registered file handle
CUfileError_t cuFileWriteAsync (CUfileHandle_t fh, void *bufPtr_base, size_t *size_p, off_t *file_offset_p, off_t *bufPtr_offset_p, ssize_t *bytes_written_p, CUstream stream)

  Variables

CUfileOpError

Detailed Description

cuFile C APIs

This file contains all the C APIs to perform GPUDirect Storage supported IO operations

Macro Definition Documentation

  #define CU_FILE_CUDA_ERR(status) ((status).cu_err)

  #define CU_FILE_RDMA_REGISTER 1

  #define CU_FILE_RDMA_RELAXED_ORDERING (1<<1)

  #define CU_FILE_STREAM_FIXED_BUF_OFFSET 1

  #define CU_FILE_STREAM_FIXED_FILE_OFFSET 2

  #define CU_FILE_STREAM_FIXED_FILE_SIZE 4

  #define CU_FILE_STREAM_PAGE_ALIGNED_INPUTS 8

  #define CUFILE_ERRSTR(err) cufileop_status_error((CUfileOpError)abs((err)))

  #define CUFILE_GPU_UUID_LEN 16

  CUfileError_t cuFileDriverClose cuFileDriverClose_v2

  #define IS_CUDA_ERR(status) ((status).err == CU_FILE_CUDA_DRIVER_ERROR)

  #define IS_CUFILE_ERR(err) (abs((err)) > CUFILEOP_BASE_ERR)

error macros to inspect error status of type CUfileOpError

Typedef Documentation

  typedef enum CUFileArrayConfigParameter_t CUFileArrayConfigParameter_t

  typedef void* CUfileBatchHandle_t

  typedef enum cufileBatchMode CUfileBatchMode_t

  typedef enum CUFileBoolConfigParameter_t CUFileBoolConfigParameter_t

  typedef struct CUfileDescr_t CUfileDescr_t

  typedef enum CUfileDriverControlFlags CUfileDriverControlFlags_t

  typedef enum CUfileDriverStatusFlags CUfileDriverStatusFlags_t

  typedef struct CUfileDrvProps CUfileDrvProps_t

  typedef struct CUfileError CUfileError_t

cufileop status string

  typedef enum CUfileFeatureFlags CUfileFeatureFlags_t

  typedef struct CUfileFSOps CUfileFSOps_t

  typedef void* CUfileHandle_t

File handle type.

  typedef struct CUfileIOEvents CUfileIOEvents_t

  typedef struct CUfileIOParams CUfileIOParams_t

  typedef enum CUfileOpcode CUfileOpcode_t

  typedef struct CUfileOpCounter CUfileOpCounter_t

Counter structure for tracking operation successes and failures.

  typedef struct CUfilePerGpuStats CUfilePerGpuStats_t

Per-GPU statistics structure used in Level 3.

  typedef struct cufileRDMAInfo cufileRDMAInfo_t

  typedef enum CUFileSizeTConfigParameter_t CUFileSizeTConfigParameter_t

  typedef struct CUfileStatsLevel1 CUfileStatsLevel1_t

Level 1 Statistics: Basic IO and operation counters.

  typedef struct CUfileStatsLevel2 CUfileStatsLevel2_t

Level 2 Statistics: Includes Level 1 plus size histograms and detailed metrics.

  typedef struct CUfileStatsLevel3 CUfileStatsLevel3_t

Level 3 Statistics: Includes Level 2 plus per-GPU and subsystem statistics.

  typedef enum CUFILEStatus_enum CUfileStatus_t

  typedef enum CUFileStringConfigParameter_t CUFileStringConfigParameter_t

  typedef struct sockaddr sockaddr_t

Enumeration Type Documentation

  enum CUFileArrayConfigParameter_t

Enumerator

CUFILE_PARAM_POSIX_POOL_SLAB_SIZE_KB

CUFILE_PARAM_POSIX_POOL_SLAB_COUNT

  enum cufileBatchMode

Enumerator

CUFILE_BATCH

  enum CUFileBoolConfigParameter_t

Enumerator

CUFILE_PARAM_PROPERTIES_USE_POLL_MODE

CUFILE_PARAM_PROPERTIES_ALLOW_COMPAT_MODE

CUFILE_PARAM_FORCE_COMPAT_MODE

CUFILE_PARAM_FS_MISC_API_CHECK_AGGRESSIVE

CUFILE_PARAM_EXECUTION_PARALLEL_IO

CUFILE_PARAM_PROFILE_NVTX

CUFILE_PARAM_PROPERTIES_ALLOW_SYSTEM_MEMORY

CUFILE_PARAM_USE_PCIP2PDMA

CUFILE_PARAM_PREFER_IO_URING

CUFILE_PARAM_FORCE_ODIRECT_MODE

CUFILE_PARAM_SKIP_TOPOLOGY_DETECTION

CUFILE_PARAM_STREAM_MEMOPS_BYPASS

  enum CUfileDriverControlFlags

Enumerator

CU_FILE_USE_POLL_MODE
    use POLL mode. properties.use_poll_mode

CU_FILE_ALLOW_COMPAT_MODE
    allow COMPATIBILITY mode. properties.allow_compat_mode

  enum CUfileDriverStatusFlags

Enumerator

CU_FILE_LUSTRE_SUPPORTED
    Support for DDN LUSTRE

CU_FILE_WEKAFS_SUPPORTED
    Support for WEKAFS

CU_FILE_NFS_SUPPORTED
    Support for NFS

CU_FILE_GPFS_SUPPORTED

CU_FILE_NVME_SUPPORTED
    < Support for GPFS Support for NVMe

CU_FILE_NVMEOF_SUPPORTED
    Support for NVMeOF

CU_FILE_SCSI_SUPPORTED
    Support for SCSI

CU_FILE_SCALEFLUX_CSD_SUPPORTED
    Support for Scaleflux CSD

CU_FILE_NVMESH_SUPPORTED
    Support for NVMesh Block Dev

CU_FILE_BEEGFS_SUPPORTED
    Support for BeeGFS

CU_FILE_NVME_P2P_SUPPORTED
    Support for NVMe using PCI P2PDMA

CU_FILE_SCATEFS_SUPPORTED
    Support for ScateFS

  enum CUfileFeatureFlags

Enumerator

CU_FILE_DYN_ROUTING_SUPPORTED
    Support for Dynamic routing to handle devices across the PCIe bridges

CU_FILE_BATCH_IO_SUPPORTED
    Unsupported

CU_FILE_STREAMS_SUPPORTED
    Unsupported

CU_FILE_PARALLEL_IO_SUPPORTED
    Unsupported

  enum CUfileFileHandleType

Enumerator

CU_FILE_HANDLE_TYPE_OPAQUE_FD
    Linux based fd

CU_FILE_HANDLE_TYPE_OPAQUE_WIN32
    Windows based handle (unsupported)

CU_FILE_HANDLE_TYPE_USERSPACE_FS

  enum CUfileOpcode

Enumerator

CUFILE_READ

CUFILE_WRITE

  enum CUFileSizeTConfigParameter_t

Enumerator

CUFILE_PARAM_PROFILE_STATS

CUFILE_PARAM_EXECUTION_MAX_IO_QUEUE_DEPTH

CUFILE_PARAM_EXECUTION_MAX_IO_THREADS

CUFILE_PARAM_EXECUTION_MIN_IO_THRESHOLD_SIZE_KB

CUFILE_PARAM_EXECUTION_MAX_REQUEST_PARALLELISM

CUFILE_PARAM_PROPERTIES_MAX_DIRECT_IO_SIZE_KB

CUFILE_PARAM_PROPERTIES_MAX_DEVICE_CACHE_SIZE_KB

CUFILE_PARAM_PROPERTIES_PER_BUFFER_CACHE_SIZE_KB

CUFILE_PARAM_PROPERTIES_MAX_DEVICE_PINNED_MEM_SIZE_KB

CUFILE_PARAM_PROPERTIES_IO_BATCHSIZE

CUFILE_PARAM_POLLTHRESHOLD_SIZE_KB

CUFILE_PARAM_PROPERTIES_BATCH_IO_TIMEOUT_MS

  enum CUFILEStatus_enum

Enumerator

CUFILE_WAITING

CUFILE_PENDING

CUFILE_INVALID

CUFILE_CANCELED

CUFILE_COMPLETE

CUFILE_TIMEOUT

CUFILE_FAILED

  enum CUFileStringConfigParameter_t

Enumerator

CUFILE_PARAM_LOGGING_LEVEL

CUFILE_PARAM_ENV_LOGFILE_PATH

CUFILE_PARAM_LOG_DIR

Function Documentation

  CUfileError_t cuFileBatchIOCancel (CUfileBatchHandle_t batch_idp)

  void cuFileBatchIODestroy (CUfileBatchHandle_t batch_idp)

  CUfileError_t cuFileBatchIOGetStatus (CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned * nr, CUfileIOEvents_t * iocbp, struct timespec * timeout)

  CUfileError_t cuFileBatchIOSetUp (CUfileBatchHandle_t * batch_idp, unsigned nr)

  CUfileError_t cuFileBatchIOSubmit (CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t * iocbp, unsigned int flags)

  CUfileError_t cuFileBufDeregister (const void * bufPtr_base)

deregister an already registered device or host memory from cuFile

Parameters
    bufPtr_base buffer pointer to deregister

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE on invalid memory pointer or unregistered memory pointer

See also
    cuFileBufRegister

    cuFileRead

    cuFileWrite

  CUfileError_t cuFileBufRegister (const void * bufPtr_base, size_t length, int flags)

register an existing cudaMalloced memory with cuFile to pin for GPUDirect Storage access or register host allocated memory with cuFile.

Parameters
    bufPtr_base buffer pointer allocated
    length size of memory region from the above specified bufPtr
    flags CU_FILE_RDMA_REGISTER

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_NVFS_DRIVER_ERROR

    CU_FILE_INVALID_VALUE

    CU_FILE_CUDA_ERROR for unsuported memory type

    CU_FILE_MEMORY_ALREADY_REGISTERED on error

    CU_FILE_GPU_MEMORY_PINNING_FAILED if not enough pinned memory is available

Note
    This memory will be use to perform GPU direct DMA from the supported storage.

Warning
    This API is intended for usecases where the memory is used as streaming buffer that is reused across multiple cuFile IO operations before calling cuFileBufDeregister

See also
    cuFileBufDeregister

    cuFileRead

    cuFileWrite

  CUfileError_t cuFileDriverClose_v2 (void)

reset the cuFile library and release the nvidia-fs driver

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_CLOSING if there are any active IO operations using cuFileRead or cuFileWrite

See also
    cuFileDriverOpen

  CUfileError_t cuFileDriverGetProperties (CUfileDrvProps_t * props)

Gets the Driver session properties.

Returns
    CU_FILE_SUCCESS on success

See also
    cuFileDriverSetPollMode

    cuFileDriverSetMaxDirectIOSize

    cuFileDriverSetMaxCacheSize

    cuFileDriverSetMaxPinnedMemSize

  CUfileError_t cuFileDriverOpen (void)

Initialize the cuFile library and open the nvidia-fs driver.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED

    CU_FILE_DRIVER_VERSION_MISMATCH on driver version mismatch error

See also
    cuFileDriverClose

  CUfileError_t cuFileDriverSetMaxCacheSize (size_t max_cache_size)

Control parameter to set maximum GPU memory reserved per device by the library for internal buffering.

Parameters
    max_cache_size The maximum GPU buffer space per device used for internal use in KB

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

    CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error

Warning
    This is an advanced command and should be tuned based on supported GPU memory

See also
    cuFileDriverGetProperties

  CUfileError_t cuFileDriverSetMaxDirectIOSize (size_t max_direct_io_size)

Control parameter to set max IO size(KB) used by the library to talk to nvidia-fs driver.

Parameters
    max_direct_io_size maximum allowed direct io size in KB

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

    CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error

Warning
    This is an advanced command and should be tuned based on available system memory

See also
    cuFileDriverGetProperties

  CUfileError_t cuFileDriverSetMaxPinnedMemSize (size_t max_pinned_size)

Sets maximum buffer space that is pinned in KB for use by cuFileBufRegister.

Parameters
    max_pinned_size maximum buffer space that is pinned in KB

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

    CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error

Warning
    This is an advanced command and should be tuned based on supported GPU memory

See also
    cuFileDriverGetProperties

  CUfileError_t cuFileDriverSetPollMode (bool poll, size_t poll_threshold_size)

Sets whether the Read/Write APIs use polling to do IO operations.

Parameters
    poll boolean to indicate whether to use poll mode or not
    poll_threshold_size max IO size to use for POLLING mode in KB

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

    CU_FILE_DRIVER_VERSION_MISMATCH, CU_FILE_DRIVER_UNSUPPORTED_LIMIT on error

Warning
    This is an advanced command and should be tuned based on available system memory

See also
    cuFileDriverGetProperties

  CUfileError_t cuFileGetBARSizeInKB (int gpuIndex, size_t * barSize)

  CUfileError_t cuFileGetParameterBool (CUFileBoolConfigParameter_t param, bool * value)

  CUfileError_t cuFileGetParameterMinMaxValue (CUFileSizeTConfigParameter_t param, size_t * min_value, size_t * max_value)

Get both the minimum and maximum settable values for a given size_t parameter in a single call.

Parameters
    param CUfile SizeT configuration parameter
    min_value Pointer to store the minimum value
    max_value Pointer to store the maximum value

Returns
    CUfileError_t Returns CU_FILE_SUCCESS on success Returns CU_FILE_INVALID_VALUE if min_value or max_value is NULL

  CUfileError_t cuFileGetParameterPosixPoolSlabArray (size_t * size_values, size_t * count_values, int len)

Get both POSIX pool slab size and count parameters as a pair.

Parameters
    size_values Buffer to receive slab sizes in KB
    count_values Buffer to receive slab counts
    len Buffer size (must match the actual parameter length)

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if inputs are invalid or buffer size doesn't match

  CUfileError_t cuFileGetParameterSizeT (CUFileSizeTConfigParameter_t param, size_t * value)

Parameters
    param The parameter to get.
    value The location where the value will be stored.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if the input parameter is invalid

Note
    If the driver is open, cuFileGetParameter*() will return the current runtime value for the given parameter.

    If the driver is not opened yet, cuFileGetParameter*() will return the currently staged value for that parameter. Staged parameter values are cleared when the driver is opened.

  CUfileError_t cuFileGetParameterString (CUFileStringConfigParameter_t param, char * desc_str, int len)

  CUfileError_t cuFileGetStatsL1 (CUfileStatsLevel1_t * stats)

Get Level 1 cuFile statistics.

Parameters
    stats Pointer to CUfileStatsLevel1_t structure to be filled

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if stats is NULL or level 1 stats not enabled

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

  CUfileError_t cuFileGetStatsL2 (CUfileStatsLevel2_t * stats)

Get Level 2 cuFile statistics.

Parameters
    stats Pointer to CUfileStatsLevel2_t structure to be filled

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if stats is NULL or level 2 stats not enabled

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

  CUfileError_t cuFileGetStatsL3 (CUfileStatsLevel3_t * stats)

Get Level 3 cuFile statistics.

Parameters
    stats Pointer to CUfileStatsLevel3_t structure to be filled

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if stats is NULL or level 3 stats not enabled

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

  CUfileError_t cuFileGetStatsLevel (int * level)

Get the current level of statistics collection for cuFile operations.

Parameters
    level Pointer to store the current statistics level

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if level is NULL

See also
    cuFileSetStatsLevel

  CUfileError_t cuFileGetVersion (int * version)

Returns
    cufile library version.

@The version is returned as (1000 major + 10 minor). @For example, CUFILE 1.7.0 would be represented by 1070.

Note
    This is useful for applications that need to inquire the library.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if the input parameter is null.

    CU_FILE_DRIVER_VERSION_READ_ERROR if the version is not available.

  void cuFileHandleDeregister (CUfileHandle_t fh)

releases a registered filehandle from cuFile

Parameters
    fh CUfileHandle_t file handle

Returns
    void

See also
    cuFileHandleRegister

  CUfileError_t cuFileHandleRegister (CUfileHandle_t * fh, CUfileDescr_t * descr)

cuFileHandleRegister is required, and performs extra checking that is memoized to provide increased performance on later cuFile operations.

Parameters
    fh CUfileHandle_t opaque file handle for IO operations
    descr CUfileDescr_t file descriptor (OS agnostic)

Returns
    CU_FILE_SUCCESS on successful completion. fh will be updated for use in cuFileRead, cuFileWrite, cuFileHandleDeregister

    CU_FILE_DRIVER_NOT_INITIALIZED on failure to load driver

    CU_FILE_IO_NOT_SUPPORTED - if filesystem is not supported

    CU_FILE_INVALID_VALUE if null or bad api arguments

    CU_FILE_INVALID_FILE_OPEN_FLAG if file is opened with unsupported modes like no O_DIRECT

    CU_FILE_INVALID_FILE_TYPE if filepath is not valid or is not a regular file

    CU_FILE_HANDLE_ALREADY_REGISTERED if file handle/descriptor is already registered

Description cuFileHandleRegister registers the open file descriptor for use with cuFile IO operations.

This API will ensure that the file's descriptor is checked for GPUDirect Storage support and returns a valid file handle on CU_FILE_SUCCESS.

Note
    the file needs to be opened in O_DIRECT mode to support GPUDirect Storage.

See also
    cuFileRead

    cuFileWrite

    cuFileHandleDeregister

  ssize_t cuFileRead (CUfileHandle_t fh, void * bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset)

read data from a registered file handle to a specified device or host memory

Parameters
    fh CUfileHandle_t opaque file handle
    bufPtr_base base address of buffer in device or host memory
    size size bytes to read
    file_offset file-offset from begining of the file
    bufPtr_offset offset relative to the bufPtr_base pointer to read into.

Returns
    size of bytes successfully read

    -1 on error, in which case errno is set to indicate filesystem errors.

    all other errors will return a negative integer value of CUfileOpError enum value.

Note
    If the bufPtr is not registered with cuFileBufRegister, the data will be buffered through preallocated pinned buffers if needed.

    This is useful for applications that need to perform IO to unaligned file offsets and/or size. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory.

See also
    cuFileBufRegister

    cuFileHandleRegister

    cuFileWrite

  CUfileError_t cuFileReadAsync (CUfileHandle_t fh, void * bufPtr_base, size_t * size_p, off_t * file_offset_p, off_t * bufPtr_offset_p, ssize_t * bytes_read_p, CUstream stream)

Parameters
    fh The cuFile handle for the file.
    bufPtr_base base address of buffer in device or host memory
    size_p pointer to size bytes to read

Note
    *size_p if the size is not known at the time of submission, then must provide the max possible size for I/O request.

Parameters
    file_offset_p pointer to file-offset from begining of the file
    bufPtr_offset_p pointer to offset relative to the bufPtr_base pointer to read into.
    bytes_read_p pointer to the number of bytes that were successfully read.
    CUstream stream cuda stream for the operation.

Returns
    size of bytes successfully read in *bytes_read_p

    -1 on error, in which case errno is set to indicate filesystem errors.

    all other errors will return a negative integer value of CUfileOpError enum value.

Note
    If the bufPtr_base is not registered with cuFileBufRegister, the data will be buffered through preallocated pinned buffers.

    This is useful for applications that need to perform IO to unaligned file offsets and/or size. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory.

    If the stream is registered with cuFileStreamRegister, the IO setup and teardown overhead will be reduced.

    on cuda stream errors, the user must call cuFileStreamDeregister to release any outstanding cuFile resources for the stream.

See also
    cuFileBufRegister

    cuFileHandleRegister

    cuFileRead

    cuFileStreamRegister

    cuFileStreamDeregister

  CUfileError_t cuFileSetParameterBool (CUFileBoolConfigParameter_t param, bool value)

  CUfileError_t cuFileSetParameterPosixPoolSlabArray (const size_t * size_values, const size_t * count_values, int len)

Set both POSIX pool slab size and count parameters as a pair.

Parameters
    size_values Array of slab sizes in KB
    count_values Array of slab counts
    len Length of both arrays (must be the same)

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if inputs are invalid

    CU_FILE_DRIVER_ALREADY_OPEN if driver is already open

  CUfileError_t cuFileSetParameterSizeT (CUFileSizeTConfigParameter_t param, size_t value)

Parameters
    param The parameter to set.
    value The source of the parameter value.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if the input parameter is inalid.

    CU_FILE_DRIVER_ALREADY_OPEN if the driver is already open.

Note
    Setting values is only permitted when the driver is not open - set parameter values are applied at driver load time.

    If the same parameter is set multiple times, only the last parameter is kept and used.

    Parameter precedence (highest to losest) is: cuFileGetParameter*() (if set), then environment variable (if exists and set), then cufile.json,

  CUfileError_t cuFileSetParameterString (CUFileStringConfigParameter_t param, const char * desc_str)

  CUfileError_t cuFileSetStatsLevel (int level)

Set the level of statistics collection for cuFile operations. This will override the cufile.json settings for stats.

Parameters
    level Statistics level (0 = disabled, 1 = basic, 2 = detailed, 3 = verbose)

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_INVALID_VALUE if level is invalid

Note
    Higher stats levels may impact performance. Level 0 disables statistics.

    Changes to stats level take effect for future operations.

See also
    cuFileGetStatsLevel

  CUfileError_t cuFileStatsReset (void)

Reset all cuFile statistics counters.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

  CUfileError_t cuFileStatsStart (void)

Start collecting cuFile statistics.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

Note
    Statistics level must be set using cuFileSetStatsLevel before calling this function

  CUfileError_t cuFileStatsStop (void)

Stop collecting cuFile statistics.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

  CUfileError_t cuFileStreamDeregister (CUstream stream)

Parameters
    CUstream cuda stream for the operation.

Note
    deallocates resources used by previous cuFile asynchronous operations for the cuda stream

    highly recommend to call after cuda stream errors to release any outstanding cuFile resources for this stream

    must be called before cuStreamDestroy call for the specified stream.

    This is useful for applications that need to perform IO to unaligned file offsets and/or size. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

    CU_FILE_INVALID_VALUE if the stream is invalid

See also
    cuFileReadAsync

    cuFileWriteAsync

    cuFileStreamRegister

  CUfileError_t cuFileStreamRegister (CUstream stream, unsigned flags)

Parameters
    CUstream cuda stream for the operation.
    flags for the stream to improve the stream execution of IO based on input parameters.

Note
    supported FLAGS are

    CU_FILE_STREAM_FIXED_BUF_OFFSET - buffer pointer offset is set at submission time

    CU_FILE_STREAM_FIXED_FILE_OFFSET - file offset is set at submission time

    CU_FILE_STREAM_FIXED_FILE_SIZE - file size is set at submission time

    CU_FILE_STREAM_PAGE_ALIGNED_INPUTS - size, offset and buffer offset are 4k aligned

    allocates resources needed to support cuFile operations asynchronously for the cuda stream

    This is useful for applications that need to perform IO to unaligned file offsets and/or size. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory.

Returns
    CU_FILE_SUCCESS on success

    CU_FILE_DRIVER_NOT_INITIALIZED if the driver is not initialized

    CU_FILE_INVALID_VALUE if the stream is invalid

See also
    cuFileReadAsync

    cuFileWriteAsync

    cuFileStreamDeregister

  long cuFileUseCount (void)

returns use count of cufile drivers at that moment by the process.

  ssize_t cuFileWrite (CUfileHandle_t fh, const void * bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset)

write data from a specified device or host memory to a registered file handle

Parameters
    fh CUfileHandle_t opaque file handle
    bufPtr_base base address of buffer in device or host memory
    size size bytes to write
    file_offset file-offset from begining of the file
    bufPtr_offset offset relative to the bufPtr_base pointer to write from.

Returns
    size of bytes successfully written

    -1 on error, in which case errno is set to indicate filesystem errors.

    all other errors will return a negative integer value of CUfileOpError enum value.

Note
    If the bufPtr is not registered with cuFileBufRegister, the data will be buffered through preallocated pinned buffers if needed.

    This is useful for applications that need to perform IO to unaligned file offsets and/or size. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory.

See also
    cuFileBufRegister

    cuFileHandleRegister

    cuFileRead

  CUfileError_t cuFileWriteAsync (CUfileHandle_t fh, void * bufPtr_base, size_t * size_p, off_t * file_offset_p, off_t * bufPtr_offset_p, ssize_t * bytes_written_p, CUstream stream)

Parameters
    fh The cuFile handle for the file.
    bufPtr_base base address of buffer in device or host memory
    size_p pointer to size bytes to write.

Note
    *size_p if the size is not known at the time of submission, then must provide the max possible size for I/O request.

Parameters
    file_offset_p pointer to file-offset from begining of the file
    bufPtr_offset_p pointer to offset relative to the bufPtr_base pointer to write from.
    bytes_written_p pointer to the number of bytes that were successfully written.
    CUstream cuda stream for the operation.

Returns
    size of bytes successfully written in *bytes_written_p

    -1 on error, in which case errno is set to indicate filesystem errors.

    all other errors will return a negative integer value of CUfileOpError enum value.

Note
    If the bufPtr_base is not registered with cuFileBufRegister, the data will be buffered through preallocated pinned buffers.

    This is useful for applications that need to perform IO to unaligned file offsets and/or size. This is also recommended for cases where the BAR1 memory size is smaller than the size of the allocated memory.

    If the stream is registered with cuFileStreamRegister prior to this call, the IO setup and teardown overhead will be reduced.

    on cuda stream errors, the user must call cuFileStreamDeregister to release any outstanding cuFile resources for the stream.

See also
    cuFileBufRegister

    cuFileHandleRegister

    cuFileWrite

    cuFileStreamRegister

    cuFileStreamDeregister

Variable Documentation

  CUfileOpError

Author

Generated automatically by Doxygen for GPUDirect Storage from the source code.