1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
7 #include <rte_common.h>
9 #include <rte_malloc.h>
10 #include <rte_errno.h>
12 #include <rte_bus_pci.h>
13 #include <rte_byteorder.h>
16 #include <gpudev_driver.h>
18 #include <cudaTypedefs.h>
20 #define CUDA_DRIVER_MIN_VERSION 11040
21 #define CUDA_API_MIN_VERSION 3020
23 /* CUDA Driver functions loaded with dlsym() */
24 static CUresult CUDAAPI (*sym_cuInit)(unsigned int flags);
25 static CUresult CUDAAPI (*sym_cuDriverGetVersion)(int *driverVersion);
26 static CUresult CUDAAPI (*sym_cuGetProcAddress)(const char *symbol,
27 void **pfn, int cudaVersion, uint64_t flags);
29 /* CUDA Driver functions loaded with cuGetProcAddress for versioning */
30 static PFN_cuGetErrorString pfn_cuGetErrorString;
31 static PFN_cuGetErrorName pfn_cuGetErrorName;
32 static PFN_cuPointerSetAttribute pfn_cuPointerSetAttribute;
33 static PFN_cuDeviceGetAttribute pfn_cuDeviceGetAttribute;
34 static PFN_cuDeviceGetByPCIBusId pfn_cuDeviceGetByPCIBusId;
35 static PFN_cuDevicePrimaryCtxRetain pfn_cuDevicePrimaryCtxRetain;
36 static PFN_cuDevicePrimaryCtxRelease pfn_cuDevicePrimaryCtxRelease;
37 static PFN_cuDeviceTotalMem pfn_cuDeviceTotalMem;
38 static PFN_cuDeviceGetName pfn_cuDeviceGetName;
39 static PFN_cuCtxGetApiVersion pfn_cuCtxGetApiVersion;
40 static PFN_cuCtxSetCurrent pfn_cuCtxSetCurrent;
41 static PFN_cuCtxGetCurrent pfn_cuCtxGetCurrent;
42 static PFN_cuCtxGetDevice pfn_cuCtxGetDevice;
43 static PFN_cuCtxGetExecAffinity pfn_cuCtxGetExecAffinity;
44 static PFN_cuMemAlloc pfn_cuMemAlloc;
45 static PFN_cuMemFree pfn_cuMemFree;
46 static PFN_cuMemHostRegister pfn_cuMemHostRegister;
47 static PFN_cuMemHostUnregister pfn_cuMemHostUnregister;
48 static PFN_cuMemHostGetDevicePointer pfn_cuMemHostGetDevicePointer;
49 static PFN_cuFlushGPUDirectRDMAWrites pfn_cuFlushGPUDirectRDMAWrites;
52 static unsigned int cuda_api_version;
53 static int cuda_driver_version;
55 /* NVIDIA GPU vendor */
56 #define NVIDIA_GPU_VENDOR_ID (0x10de)
58 /* NVIDIA GPU device IDs */
59 #define NVIDIA_GPU_A100_40GB_DEVICE_ID (0x20f1)
60 #define NVIDIA_GPU_A100_80GB_DEVICE_ID (0x20b5)
61 #define NVIDIA_GPU_A100_80GB_DPU_DEVICE_ID (0x20b8)
63 #define NVIDIA_GPU_A30_24GB_DEVICE_ID (0x20b7)
64 #define NVIDIA_GPU_A10_24GB_DEVICE_ID (0x2236)
66 #define NVIDIA_GPU_V100_32GB_SXM_DEVICE_ID (0x1db5)
67 #define NVIDIA_GPU_V100_32GB_PCIE_DEVICE_ID (0x1db6)
68 #define NVIDIA_GPU_V100_16GB_DEVICE_ID (0x1db4)
70 #define NVIDIA_GPU_T4_16GB_DEVICE_ID (0x1eb8)
72 #define CUDA_MAX_ALLOCATION_NUM 512
74 #define GPU_PAGE_SHIFT 16
75 #define GPU_PAGE_SIZE (1UL << GPU_PAGE_SHIFT)
77 static RTE_LOG_REGISTER_DEFAULT(cuda_logtype, NOTICE);
79 /* Helper macro for logging */
80 #define rte_cuda_log(level, fmt, ...) \
81 rte_log(RTE_LOG_ ## level, cuda_logtype, fmt "\n", ##__VA_ARGS__)
83 #define rte_cuda_debug(fmt, ...) \
84 rte_cuda_log(DEBUG, RTE_STR(__LINE__) ":%s() " fmt, __func__, \
87 /* NVIDIA GPU address map */
88 static const struct rte_pci_id pci_id_cuda_map[] = {
90 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
91 NVIDIA_GPU_A100_40GB_DEVICE_ID)
94 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
95 NVIDIA_GPU_A100_80GB_DEVICE_ID)
98 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
99 NVIDIA_GPU_A100_80GB_DPU_DEVICE_ID)
102 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
103 NVIDIA_GPU_A30_24GB_DEVICE_ID)
106 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
107 NVIDIA_GPU_A10_24GB_DEVICE_ID)
110 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
111 NVIDIA_GPU_V100_32GB_SXM_DEVICE_ID)
114 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
115 NVIDIA_GPU_V100_32GB_PCIE_DEVICE_ID)
118 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
119 NVIDIA_GPU_V100_16GB_DEVICE_ID)
122 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
123 NVIDIA_GPU_T4_16GB_DEVICE_ID)
130 /* Device private info */
132 char gpu_name[RTE_DEV_NAME_MAX_LEN];
135 int gdr_write_ordering;
139 /* Type of memory allocated by CUDA driver */
143 GPU_REGISTERED /* Not used yet */
146 /* key associated to a memory address */
147 typedef uintptr_t cuda_ptr_key;
149 /* Single entry of the memory list */
152 CUdeviceptr ptr_orig_d;
160 struct mem_entry *prev;
161 struct mem_entry *next;
164 static struct mem_entry *mem_alloc_list_head;
165 static struct mem_entry *mem_alloc_list_tail;
166 static uint32_t mem_alloc_list_last_elem;
168 /* Load the CUDA symbols */
173 char cuda_path[1024];
175 if (getenv("CUDA_PATH_L") == NULL)
176 snprintf(cuda_path, 1024, "%s", "libcuda.so");
178 snprintf(cuda_path, 1024, "%s%s", getenv("CUDA_PATH_L"), "libcuda.so");
180 cudalib = dlopen(cuda_path, RTLD_LAZY);
181 if (cudalib == NULL) {
182 rte_cuda_log(ERR, "Failed to find CUDA library in %s (CUDA_PATH_L=%s)",
183 cuda_path, getenv("CUDA_PATH_L"));
191 cuda_sym_func_loader(void)
196 sym_cuInit = dlsym(cudalib, "cuInit");
197 if (sym_cuInit == NULL) {
198 rte_cuda_log(ERR, "Failed to load CUDA missing symbol cuInit");
202 sym_cuDriverGetVersion = dlsym(cudalib, "cuDriverGetVersion");
203 if (sym_cuDriverGetVersion == NULL) {
204 rte_cuda_log(ERR, "Failed to load CUDA missing symbol cuDriverGetVersion");
208 sym_cuGetProcAddress = dlsym(cudalib, "cuGetProcAddress");
209 if (sym_cuGetProcAddress == NULL) {
210 rte_cuda_log(ERR, "Failed to load CUDA missing symbol cuGetProcAddress");
218 cuda_pfn_func_loader(void)
222 res = sym_cuGetProcAddress("cuGetErrorString",
223 (void **) (&pfn_cuGetErrorString), cuda_driver_version, 0);
225 rte_cuda_log(ERR, "Retrieve pfn_cuGetErrorString failed with %d", res);
229 res = sym_cuGetProcAddress("cuGetErrorName",
230 (void **)(&pfn_cuGetErrorName), cuda_driver_version, 0);
232 rte_cuda_log(ERR, "Retrieve pfn_cuGetErrorName failed with %d", res);
236 res = sym_cuGetProcAddress("cuPointerSetAttribute",
237 (void **)(&pfn_cuPointerSetAttribute), cuda_driver_version, 0);
239 rte_cuda_log(ERR, "Retrieve pfn_cuPointerSetAttribute failed with %d", res);
243 res = sym_cuGetProcAddress("cuDeviceGetAttribute",
244 (void **)(&pfn_cuDeviceGetAttribute), cuda_driver_version, 0);
246 rte_cuda_log(ERR, "Retrieve pfn_cuDeviceGetAttribute failed with %d", res);
250 res = sym_cuGetProcAddress("cuDeviceGetByPCIBusId",
251 (void **)(&pfn_cuDeviceGetByPCIBusId), cuda_driver_version, 0);
253 rte_cuda_log(ERR, "Retrieve pfn_cuDeviceGetByPCIBusId failed with %d", res);
257 res = sym_cuGetProcAddress("cuDeviceGetName",
258 (void **)(&pfn_cuDeviceGetName), cuda_driver_version, 0);
260 rte_cuda_log(ERR, "Retrieve pfn_cuDeviceGetName failed with %d", res);
264 res = sym_cuGetProcAddress("cuDevicePrimaryCtxRetain",
265 (void **)(&pfn_cuDevicePrimaryCtxRetain), cuda_driver_version, 0);
267 rte_cuda_log(ERR, "Retrieve pfn_cuDevicePrimaryCtxRetain failed with %d", res);
271 res = sym_cuGetProcAddress("cuDevicePrimaryCtxRelease",
272 (void **)(&pfn_cuDevicePrimaryCtxRelease), cuda_driver_version, 0);
274 rte_cuda_log(ERR, "Retrieve pfn_cuDevicePrimaryCtxRelease failed with %d", res);
278 res = sym_cuGetProcAddress("cuDeviceTotalMem",
279 (void **)(&pfn_cuDeviceTotalMem), cuda_driver_version, 0);
281 rte_cuda_log(ERR, "Retrieve pfn_cuDeviceTotalMem failed with %d", res);
285 res = sym_cuGetProcAddress("cuCtxGetApiVersion",
286 (void **)(&pfn_cuCtxGetApiVersion), cuda_driver_version, 0);
288 rte_cuda_log(ERR, "Retrieve pfn_cuCtxGetApiVersion failed with %d", res);
292 res = sym_cuGetProcAddress("cuCtxGetDevice",
293 (void **)(&pfn_cuCtxGetDevice), cuda_driver_version, 0);
295 rte_cuda_log(ERR, "Retrieve pfn_cuCtxGetDevice failed with %d", res);
299 res = sym_cuGetProcAddress("cuCtxSetCurrent",
300 (void **)(&pfn_cuCtxSetCurrent), cuda_driver_version, 0);
302 rte_cuda_log(ERR, "Retrieve pfn_cuCtxSetCurrent failed with %d", res);
306 res = sym_cuGetProcAddress("cuCtxGetCurrent",
307 (void **)(&pfn_cuCtxGetCurrent), cuda_driver_version, 0);
309 rte_cuda_log(ERR, "Retrieve pfn_cuCtxGetCurrent failed with %d", res);
313 res = sym_cuGetProcAddress("cuCtxGetExecAffinity",
314 (void **)(&pfn_cuCtxGetExecAffinity), cuda_driver_version, 0);
316 rte_cuda_log(ERR, "Retrieve pfn_cuCtxGetExecAffinity failed with %d", res);
320 res = sym_cuGetProcAddress("cuMemAlloc",
321 (void **)(&pfn_cuMemAlloc), cuda_driver_version, 0);
323 rte_cuda_log(ERR, "Retrieve pfn_cuMemAlloc failed with %d", res);
327 res = sym_cuGetProcAddress("cuMemFree",
328 (void **)(&pfn_cuMemFree), cuda_driver_version, 0);
330 rte_cuda_log(ERR, "Retrieve pfn_cuMemFree failed with %d", res);
334 res = sym_cuGetProcAddress("cuMemHostRegister",
335 (void **)(&pfn_cuMemHostRegister), cuda_driver_version, 0);
337 rte_cuda_log(ERR, "Retrieve pfn_cuMemHostRegister failed with %d", res);
341 res = sym_cuGetProcAddress("cuMemHostUnregister",
342 (void **)(&pfn_cuMemHostUnregister), cuda_driver_version, 0);
344 rte_cuda_log(ERR, "Retrieve pfn_cuMemHostUnregister failed with %d", res);
348 res = sym_cuGetProcAddress("cuMemHostGetDevicePointer",
349 (void **)(&pfn_cuMemHostGetDevicePointer), cuda_driver_version, 0);
351 rte_cuda_log(ERR, "Retrieve pfn_cuMemHostGetDevicePointer failed with %d", res);
355 res = sym_cuGetProcAddress("cuFlushGPUDirectRDMAWrites",
356 (void **)(&pfn_cuFlushGPUDirectRDMAWrites), cuda_driver_version, 0);
358 rte_cuda_log(ERR, "Retrieve cuFlushGPUDirectRDMAWrites failed with %d", res);
365 /* Generate a key from a memory pointer */
367 get_hash_from_ptr(void *ptr)
369 return (uintptr_t)ptr;
373 mem_list_count_item(void)
375 return mem_alloc_list_last_elem;
378 /* Initiate list of memory allocations if not done yet */
379 static struct mem_entry *
380 mem_list_add_item(void)
382 /* Initiate list of memory allocations if not done yet */
383 if (mem_alloc_list_head == NULL) {
384 mem_alloc_list_head = rte_zmalloc(NULL,
385 sizeof(struct mem_entry),
386 RTE_CACHE_LINE_SIZE);
387 if (mem_alloc_list_head == NULL) {
388 rte_cuda_log(ERR, "Failed to allocate memory for memory list");
392 mem_alloc_list_head->next = NULL;
393 mem_alloc_list_head->prev = NULL;
394 mem_alloc_list_tail = mem_alloc_list_head;
396 struct mem_entry *mem_alloc_list_cur = rte_zmalloc(NULL,
397 sizeof(struct mem_entry),
398 RTE_CACHE_LINE_SIZE);
400 if (mem_alloc_list_cur == NULL) {
401 rte_cuda_log(ERR, "Failed to allocate memory for memory list");
405 mem_alloc_list_tail->next = mem_alloc_list_cur;
406 mem_alloc_list_cur->prev = mem_alloc_list_tail;
407 mem_alloc_list_tail = mem_alloc_list_tail->next;
408 mem_alloc_list_tail->next = NULL;
411 mem_alloc_list_last_elem++;
413 return mem_alloc_list_tail;
416 static struct mem_entry *
417 mem_list_find_item(cuda_ptr_key pk)
419 struct mem_entry *mem_alloc_list_cur = NULL;
421 if (mem_alloc_list_head == NULL) {
422 rte_cuda_log(ERR, "Memory list doesn't exist");
426 if (mem_list_count_item() == 0) {
427 rte_cuda_log(ERR, "No items in memory list");
431 mem_alloc_list_cur = mem_alloc_list_head;
433 while (mem_alloc_list_cur != NULL) {
434 if (mem_alloc_list_cur->pkey == pk)
435 return mem_alloc_list_cur;
436 mem_alloc_list_cur = mem_alloc_list_cur->next;
439 return mem_alloc_list_cur;
443 mem_list_del_item(cuda_ptr_key pk)
445 struct mem_entry *mem_alloc_list_cur = NULL;
447 mem_alloc_list_cur = mem_list_find_item(pk);
448 if (mem_alloc_list_cur == NULL)
451 /* if key is in head */
452 if (mem_alloc_list_cur->prev == NULL) {
453 mem_alloc_list_head = mem_alloc_list_cur->next;
454 if (mem_alloc_list_head != NULL)
455 mem_alloc_list_head->prev = NULL;
457 mem_alloc_list_cur->prev->next = mem_alloc_list_cur->next;
458 if (mem_alloc_list_cur->next != NULL)
459 mem_alloc_list_cur->next->prev = mem_alloc_list_cur->prev;
462 rte_free(mem_alloc_list_cur);
464 mem_alloc_list_last_elem--;
470 cuda_dev_info_get(struct rte_gpu *dev, struct rte_gpu_info *info)
474 struct rte_gpu_info parent_info;
475 CUexecAffinityParam affinityPrm;
476 const char *err_string;
477 struct cuda_info *private;
478 CUcontext current_ctx;
486 /* Child initialization time probably called by rte_gpu_add_child() */
487 if (dev->mpshared->info.parent != RTE_GPU_ID_NONE &&
488 dev->mpshared->dev_private == NULL) {
489 /* Store current ctx */
490 res = pfn_cuCtxGetCurrent(¤t_ctx);
492 pfn_cuGetErrorString(res, &(err_string));
493 rte_cuda_log(ERR, "cuCtxGetCurrent failed with %s",
499 /* Set child ctx as current ctx */
500 input_ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
501 res = pfn_cuCtxSetCurrent(input_ctx);
503 pfn_cuGetErrorString(res, &(err_string));
504 rte_cuda_log(ERR, "cuCtxSetCurrent input failed with %s",
515 res = pfn_cuCtxGetExecAffinity(&affinityPrm,
516 CU_EXEC_AFFINITY_TYPE_SM_COUNT);
518 pfn_cuGetErrorString(res, &(err_string));
519 rte_cuda_log(ERR, "cuCtxGetExecAffinity failed with %s",
522 dev->mpshared->info.processor_count =
523 (uint32_t)affinityPrm.param.smCount.val;
525 ret = rte_gpu_info_get(dev->mpshared->info.parent, &parent_info);
530 dev->mpshared->info.total_memory = parent_info.total_memory;
533 * GPU Device private info
535 dev->mpshared->dev_private = rte_zmalloc(NULL,
536 sizeof(struct cuda_info),
537 RTE_CACHE_LINE_SIZE);
538 if (dev->mpshared->dev_private == NULL) {
539 rte_cuda_log(ERR, "Failed to allocate memory for GPU process private");
544 private = (struct cuda_info *)dev->mpshared->dev_private;
546 res = pfn_cuCtxGetDevice(&(private->cu_dev));
548 pfn_cuGetErrorString(res, &(err_string));
549 rte_cuda_log(ERR, "cuCtxGetDevice failed with %s",
555 res = pfn_cuDeviceGetName(private->gpu_name,
556 RTE_DEV_NAME_MAX_LEN, private->cu_dev);
558 pfn_cuGetErrorString(res, &(err_string));
559 rte_cuda_log(ERR, "cuDeviceGetName failed with %s",
565 /* Restore original ctx as current ctx */
566 res = pfn_cuCtxSetCurrent(current_ctx);
568 pfn_cuGetErrorString(res, &(err_string));
569 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
576 *info = dev->mpshared->info;
586 cuda_mem_alloc(struct rte_gpu *dev, size_t size, unsigned int align, void **ptr)
589 const char *err_string;
590 CUcontext current_ctx;
592 unsigned int flag = 1;
597 /* Store current ctx */
598 res = pfn_cuCtxGetCurrent(¤t_ctx);
600 pfn_cuGetErrorString(res, &(err_string));
601 rte_cuda_log(ERR, "cuCtxGetCurrent failed with %s",
607 /* Set child ctx as current ctx */
608 input_ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
609 res = pfn_cuCtxSetCurrent(input_ctx);
611 pfn_cuGetErrorString(res, &(err_string));
612 rte_cuda_log(ERR, "cuCtxSetCurrent input failed with %s",
618 /* Get next memory list item */
619 mem_alloc_list_tail = mem_list_add_item();
620 if (mem_alloc_list_tail == NULL) {
625 /* Allocate memory */
626 mem_alloc_list_tail->size = size;
627 mem_alloc_list_tail->size_orig = size + align;
629 res = pfn_cuMemAlloc(&(mem_alloc_list_tail->ptr_orig_d),
630 mem_alloc_list_tail->size_orig);
632 pfn_cuGetErrorString(res, &(err_string));
633 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
639 /* Align memory address */
640 mem_alloc_list_tail->ptr_d = mem_alloc_list_tail->ptr_orig_d;
641 if (align && ((uintptr_t)mem_alloc_list_tail->ptr_d) % align)
642 mem_alloc_list_tail->ptr_d += (align -
643 (((uintptr_t)mem_alloc_list_tail->ptr_d) % align));
645 /* GPUDirect RDMA attribute required */
646 res = pfn_cuPointerSetAttribute(&flag,
647 CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
648 mem_alloc_list_tail->ptr_d);
650 rte_cuda_log(ERR, "Could not set SYNC MEMOP attribute for "
651 "GPU memory at %"PRIu32", err %d",
652 (uint32_t)mem_alloc_list_tail->ptr_d, res);
657 mem_alloc_list_tail->pkey = get_hash_from_ptr((void *)mem_alloc_list_tail->ptr_d);
658 mem_alloc_list_tail->ptr_h = NULL;
659 mem_alloc_list_tail->dev = dev;
660 mem_alloc_list_tail->ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
661 mem_alloc_list_tail->mtype = GPU_MEM;
663 /* Restore original ctx as current ctx */
664 res = pfn_cuCtxSetCurrent(current_ctx);
666 pfn_cuGetErrorString(res, &(err_string));
667 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
673 *ptr = (void *)mem_alloc_list_tail->ptr_d;
679 cuda_mem_register(struct rte_gpu *dev, size_t size, void *ptr)
682 const char *err_string;
683 CUcontext current_ctx;
685 unsigned int flag = 1;
691 /* Store current ctx */
692 res = pfn_cuCtxGetCurrent(¤t_ctx);
694 pfn_cuGetErrorString(res, &(err_string));
695 rte_cuda_log(ERR, "cuCtxGetCurrent failed with %s",
701 /* Set child ctx as current ctx */
702 input_ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
703 res = pfn_cuCtxSetCurrent(input_ctx);
705 pfn_cuGetErrorString(res, &(err_string));
706 rte_cuda_log(ERR, "cuCtxSetCurrent input failed with %s",
712 /* Get next memory list item */
713 mem_alloc_list_tail = mem_list_add_item();
714 if (mem_alloc_list_tail == NULL) {
719 /* Allocate memory */
720 mem_alloc_list_tail->size = size;
721 mem_alloc_list_tail->ptr_h = ptr;
723 res = pfn_cuMemHostRegister(mem_alloc_list_tail->ptr_h,
724 mem_alloc_list_tail->size,
725 CU_MEMHOSTREGISTER_PORTABLE |
726 CU_MEMHOSTREGISTER_DEVICEMAP);
728 pfn_cuGetErrorString(res, &(err_string));
729 rte_cuda_log(ERR, "cuMemHostRegister failed with %s ptr %p size %zd",
731 mem_alloc_list_tail->ptr_h,
732 mem_alloc_list_tail->size);
737 res = pfn_cuDeviceGetAttribute(&(use_ptr_h),
738 CU_DEVICE_ATTRIBUTE_CAN_USE_HOST_POINTER_FOR_REGISTERED_MEM,
739 ((struct cuda_info *)(dev->mpshared->dev_private))->cu_dev);
741 pfn_cuGetErrorString(res, &(err_string));
742 rte_cuda_log(ERR, "cuDeviceGetAttribute failed with %s",
748 if (use_ptr_h == 0) {
749 res = pfn_cuMemHostGetDevicePointer(&(mem_alloc_list_tail->ptr_d),
750 mem_alloc_list_tail->ptr_h, 0);
752 pfn_cuGetErrorString(res, &(err_string));
753 rte_cuda_log(ERR, "cuMemHostGetDevicePointer failed with %s",
759 if ((uintptr_t)mem_alloc_list_tail->ptr_d !=
760 (uintptr_t)mem_alloc_list_tail->ptr_h) {
761 rte_cuda_log(ERR, "Host input pointer is different wrt GPU registered pointer");
766 mem_alloc_list_tail->ptr_d = (CUdeviceptr)mem_alloc_list_tail->ptr_h;
769 /* GPUDirect RDMA attribute required */
770 res = pfn_cuPointerSetAttribute(&flag,
771 CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
772 mem_alloc_list_tail->ptr_d);
774 rte_cuda_log(ERR, "Could not set SYNC MEMOP attribute for GPU memory at %"PRIu32
775 ", err %d", (uint32_t)mem_alloc_list_tail->ptr_d, res);
780 mem_alloc_list_tail->pkey = get_hash_from_ptr((void *)mem_alloc_list_tail->ptr_h);
781 mem_alloc_list_tail->size = size;
782 mem_alloc_list_tail->dev = dev;
783 mem_alloc_list_tail->ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
784 mem_alloc_list_tail->mtype = CPU_REGISTERED;
785 mem_alloc_list_tail->ptr_orig_d = mem_alloc_list_tail->ptr_d;
787 /* Restore original ctx as current ctx */
788 res = pfn_cuCtxSetCurrent(current_ctx);
790 pfn_cuGetErrorString(res, &(err_string));
791 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
801 cuda_mem_free(struct rte_gpu *dev, void *ptr)
804 struct mem_entry *mem_item;
805 const char *err_string;
811 hk = get_hash_from_ptr((void *)ptr);
813 mem_item = mem_list_find_item(hk);
814 if (mem_item == NULL) {
815 rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory", ptr);
820 if (mem_item->mtype == GPU_MEM) {
821 res = pfn_cuMemFree(mem_item->ptr_orig_d);
823 pfn_cuGetErrorString(res, &(err_string));
824 rte_cuda_log(ERR, "cuMemFree current failed with %s",
830 return mem_list_del_item(hk);
833 rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
839 cuda_mem_unregister(struct rte_gpu *dev, void *ptr)
842 struct mem_entry *mem_item;
843 const char *err_string;
849 hk = get_hash_from_ptr((void *)ptr);
851 mem_item = mem_list_find_item(hk);
852 if (mem_item == NULL) {
853 rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory", ptr);
858 if (mem_item->mtype == CPU_REGISTERED) {
859 res = pfn_cuMemHostUnregister(ptr);
861 pfn_cuGetErrorString(res, &(err_string));
862 rte_cuda_log(ERR, "cuMemHostUnregister current failed with %s",
868 return mem_list_del_item(hk);
871 rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
878 cuda_dev_close(struct rte_gpu *dev)
883 rte_free(dev->mpshared->dev_private);
889 cuda_wmb(struct rte_gpu *dev)
892 const char *err_string;
893 CUcontext current_ctx;
895 struct cuda_info *private;
902 private = (struct cuda_info *)dev->mpshared->dev_private;
904 if (private->gdr_write_ordering != CU_GPU_DIRECT_RDMA_WRITES_ORDERING_NONE) {
906 * No need to explicitly force the write ordering because
907 * the device natively supports it
912 if (private->gdr_flush_type != CU_FLUSH_GPU_DIRECT_RDMA_WRITES_OPTION_HOST) {
914 * Can't flush GDR writes with cuFlushGPUDirectRDMAWrites CUDA function.
915 * Application needs to use alternative methods.
917 rte_cuda_log(WARNING, "Can't flush GDR writes with cuFlushGPUDirectRDMAWrites CUDA function."
918 "Application needs to use alternative methods.");
924 /* Store current ctx */
925 res = pfn_cuCtxGetCurrent(¤t_ctx);
927 pfn_cuGetErrorString(res, &(err_string));
928 rte_cuda_log(ERR, "cuCtxGetCurrent failed with %s",
934 /* Set child ctx as current ctx */
935 input_ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
936 res = pfn_cuCtxSetCurrent(input_ctx);
938 pfn_cuGetErrorString(res, &(err_string));
939 rte_cuda_log(ERR, "cuCtxSetCurrent input failed with %s",
945 res = pfn_cuFlushGPUDirectRDMAWrites(CU_FLUSH_GPU_DIRECT_RDMA_WRITES_TARGET_CURRENT_CTX,
946 CU_FLUSH_GPU_DIRECT_RDMA_WRITES_TO_ALL_DEVICES);
948 pfn_cuGetErrorString(res, &(err_string));
949 rte_cuda_log(ERR, "cuFlushGPUDirectRDMAWrites current failed with %s",
955 /* Restore original ctx as current ctx */
956 res = pfn_cuCtxSetCurrent(current_ctx);
958 pfn_cuGetErrorString(res, &(err_string));
959 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
969 cuda_gpu_probe(__rte_unused struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
971 struct rte_gpu *dev = NULL;
975 char dev_name[RTE_DEV_NAME_MAX_LEN];
976 const char *err_string;
977 int processor_count = 0;
978 struct cuda_info *private;
980 if (pci_dev == NULL) {
981 rte_cuda_log(ERR, "NULL PCI device");
986 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
988 /* Allocate memory to be used privately by drivers */
989 dev = rte_gpu_allocate(pci_dev->device.name);
995 /* Initialize values only for the first CUDA driver call */
996 if (dev->mpshared->info.dev_id == 0) {
997 mem_alloc_list_head = NULL;
998 mem_alloc_list_tail = NULL;
999 mem_alloc_list_last_elem = 0;
1001 /* Load libcuda.so library */
1002 if (cuda_loader()) {
1003 rte_cuda_log(ERR, "CUDA Driver library not found");
1004 rte_errno = ENOTSUP;
1008 /* Load initial CUDA functions */
1009 if (cuda_sym_func_loader()) {
1010 rte_cuda_log(ERR, "CUDA functions not found in library");
1011 rte_errno = ENOTSUP;
1016 * Required to initialize the CUDA Driver.
1017 * Multiple calls of cuInit() will return immediately
1018 * without making any relevant change
1022 res = sym_cuDriverGetVersion(&cuda_driver_version);
1024 rte_cuda_log(ERR, "cuDriverGetVersion failed with %d", res);
1025 rte_errno = ENOTSUP;
1029 if (cuda_driver_version < CUDA_DRIVER_MIN_VERSION) {
1030 rte_cuda_log(ERR, "CUDA Driver version found is %d. "
1031 "Minimum requirement is %d",
1032 cuda_driver_version,
1033 CUDA_DRIVER_MIN_VERSION);
1034 rte_errno = ENOTSUP;
1038 if (cuda_pfn_func_loader()) {
1039 rte_cuda_log(ERR, "CUDA PFN functions not found in library");
1040 rte_errno = ENOTSUP;
1045 /* Fill HW specific part of device structure */
1046 dev->device = &pci_dev->device;
1047 dev->mpshared->info.numa_node = pci_dev->device.numa_node;
1049 /* Get NVIDIA GPU Device descriptor */
1050 res = pfn_cuDeviceGetByPCIBusId(&cu_dev_id, dev->device->name);
1052 pfn_cuGetErrorString(res, &(err_string));
1053 rte_cuda_log(ERR, "cuDeviceGetByPCIBusId name %s failed with %d: %s",
1054 dev->device->name, res, err_string);
1059 res = pfn_cuDevicePrimaryCtxRetain(&pctx, cu_dev_id);
1061 pfn_cuGetErrorString(res, &(err_string));
1062 rte_cuda_log(ERR, "cuDevicePrimaryCtxRetain name %s failed with %d: %s",
1063 dev->device->name, res, err_string);
1068 res = pfn_cuCtxGetApiVersion(pctx, &cuda_api_version);
1070 rte_cuda_log(ERR, "cuCtxGetApiVersion failed with %d", res);
1071 rte_errno = ENOTSUP;
1075 if (cuda_api_version < CUDA_API_MIN_VERSION) {
1076 rte_cuda_log(ERR, "CUDA API version found is %d Minimum requirement is %d",
1077 cuda_api_version, CUDA_API_MIN_VERSION);
1078 rte_errno = ENOTSUP;
1082 dev->mpshared->info.context = (uint64_t)pctx;
1085 * GPU Device generic info
1088 /* Processor count */
1089 res = pfn_cuDeviceGetAttribute(&(processor_count),
1090 CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT,
1093 pfn_cuGetErrorString(res, &(err_string));
1094 rte_cuda_log(ERR, "cuDeviceGetAttribute failed with %s",
1099 dev->mpshared->info.processor_count = (uint32_t)processor_count;
1102 res = pfn_cuDeviceTotalMem(&dev->mpshared->info.total_memory, cu_dev_id);
1104 pfn_cuGetErrorString(res, &(err_string));
1105 rte_cuda_log(ERR, "cuDeviceTotalMem failed with %s",
1112 * GPU Device private info
1114 dev->mpshared->dev_private = rte_zmalloc(NULL,
1115 sizeof(struct cuda_info),
1116 RTE_CACHE_LINE_SIZE);
1117 if (dev->mpshared->dev_private == NULL) {
1118 rte_cuda_log(ERR, "Failed to allocate memory for GPU process private");
1123 private = (struct cuda_info *)dev->mpshared->dev_private;
1124 private->cu_dev = cu_dev_id;
1125 res = pfn_cuDeviceGetName(private->gpu_name,
1126 RTE_DEV_NAME_MAX_LEN,
1129 pfn_cuGetErrorString(res, &(err_string));
1130 rte_cuda_log(ERR, "cuDeviceGetName failed with %s",
1136 res = pfn_cuDeviceGetAttribute(&(private->gdr_supported),
1137 CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_SUPPORTED,
1140 pfn_cuGetErrorString(res, &(err_string));
1141 rte_cuda_log(ERR, "cuDeviceGetAttribute failed with %s",
1147 if (private->gdr_supported == 0)
1148 rte_cuda_log(WARNING, "GPU %s doesn't support GPUDirect RDMA",
1149 pci_dev->device.name);
1151 res = pfn_cuDeviceGetAttribute(&(private->gdr_write_ordering),
1152 CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WRITES_ORDERING,
1155 pfn_cuGetErrorString(res, &(err_string));
1157 "cuDeviceGetAttribute failed with %s",
1163 if (private->gdr_write_ordering == CU_GPU_DIRECT_RDMA_WRITES_ORDERING_NONE) {
1164 res = pfn_cuDeviceGetAttribute(&(private->gdr_flush_type),
1165 CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_FLUSH_WRITES_OPTIONS,
1168 pfn_cuGetErrorString(res, &(err_string));
1169 rte_cuda_log(ERR, "cuDeviceGetAttribute failed with %s",
1175 if (private->gdr_flush_type != CU_FLUSH_GPU_DIRECT_RDMA_WRITES_OPTION_HOST)
1176 rte_cuda_log(ERR, "GPUDirect RDMA flush writes API is not supported");
1179 dev->ops.dev_info_get = cuda_dev_info_get;
1180 dev->ops.dev_close = cuda_dev_close;
1181 dev->ops.mem_alloc = cuda_mem_alloc;
1182 dev->ops.mem_free = cuda_mem_free;
1183 dev->ops.mem_register = cuda_mem_register;
1184 dev->ops.mem_unregister = cuda_mem_unregister;
1185 dev->ops.mem_cpu_map = NULL;
1186 dev->ops.mem_cpu_unmap = NULL;
1187 dev->ops.wmb = cuda_wmb;
1189 rte_gpu_complete_new(dev);
1191 rte_cuda_debug("dev id = %u name = %s",
1192 dev->mpshared->info.dev_id, private->gpu_name);
1198 cuda_gpu_remove(struct rte_pci_device *pci_dev)
1200 struct rte_gpu *dev;
1204 if (pci_dev == NULL) {
1209 dev = rte_gpu_get_by_name(pci_dev->device.name);
1211 rte_cuda_log(ERR, "Couldn't find HW dev \"%s\" to uninitialise it",
1212 pci_dev->device.name);
1216 gpu_id = dev->mpshared->info.dev_id;
1218 /* release dev from library */
1219 ret = rte_gpu_release(dev);
1221 rte_cuda_log(ERR, "Device %i failed to uninit: %i", gpu_id, ret);
1223 rte_cuda_debug("Destroyed dev = %u", gpu_id);
1228 static struct rte_pci_driver rte_cuda_driver = {
1229 .id_table = pci_id_cuda_map,
1230 .drv_flags = RTE_PCI_DRV_WC_ACTIVATE,
1231 .probe = cuda_gpu_probe,
1232 .remove = cuda_gpu_remove,
1235 RTE_PMD_REGISTER_PCI(gpu_cuda, rte_cuda_driver);
1236 RTE_PMD_REGISTER_PCI_TABLE(gpu_cuda, pci_id_cuda_map);
1237 RTE_PMD_REGISTER_KMOD_DEP(gpu_cuda, "* nvidia & (nv_peer_mem | nvpeer_mem)");