1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
7 #include <rte_malloc.h>
9 #include <rte_bus_pci.h>
10 #include <rte_byteorder.h>
13 #include <gpudev_driver.h>
16 #include <cudaTypedefs.h>
21 #define CUDA_DRIVER_MIN_VERSION 11040
22 #define CUDA_API_MIN_VERSION 3020
24 /* CUDA Driver functions loaded with dlsym() */
25 static CUresult CUDAAPI (*sym_cuInit)(unsigned int flags);
26 static CUresult CUDAAPI (*sym_cuDriverGetVersion)(int *driverVersion);
27 static CUresult CUDAAPI (*sym_cuGetProcAddress)(const char *symbol,
28 void **pfn, int cudaVersion, uint64_t flags);
30 /* CUDA Driver functions loaded with cuGetProcAddress for versioning */
31 static PFN_cuGetErrorString pfn_cuGetErrorString;
32 static PFN_cuGetErrorName pfn_cuGetErrorName;
33 static PFN_cuPointerSetAttribute pfn_cuPointerSetAttribute;
34 static PFN_cuDeviceGetAttribute pfn_cuDeviceGetAttribute;
35 static PFN_cuDeviceGetByPCIBusId pfn_cuDeviceGetByPCIBusId;
36 static PFN_cuDevicePrimaryCtxRetain pfn_cuDevicePrimaryCtxRetain;
37 static PFN_cuDevicePrimaryCtxRelease pfn_cuDevicePrimaryCtxRelease;
38 static PFN_cuDeviceTotalMem pfn_cuDeviceTotalMem;
39 static PFN_cuDeviceGetName pfn_cuDeviceGetName;
40 static PFN_cuCtxGetApiVersion pfn_cuCtxGetApiVersion;
41 static PFN_cuCtxSetCurrent pfn_cuCtxSetCurrent;
42 static PFN_cuCtxGetCurrent pfn_cuCtxGetCurrent;
43 static PFN_cuCtxGetDevice pfn_cuCtxGetDevice;
44 static PFN_cuCtxGetExecAffinity pfn_cuCtxGetExecAffinity;
45 static PFN_cuMemAlloc pfn_cuMemAlloc;
46 static PFN_cuMemFree pfn_cuMemFree;
47 static PFN_cuMemHostRegister pfn_cuMemHostRegister;
48 static PFN_cuMemHostUnregister pfn_cuMemHostUnregister;
49 static PFN_cuMemHostGetDevicePointer pfn_cuMemHostGetDevicePointer;
50 static PFN_cuFlushGPUDirectRDMAWrites pfn_cuFlushGPUDirectRDMAWrites;
53 static unsigned int cuda_api_version;
54 static int cuda_driver_version;
57 #define CUDA_MAX_ALLOCATION_NUM 512
59 #define GPU_PAGE_SHIFT 16
60 #define GPU_PAGE_SIZE (1UL << GPU_PAGE_SHIFT)
62 RTE_LOG_REGISTER_DEFAULT(cuda_logtype, NOTICE);
64 /* NVIDIA GPU address map */
65 static const struct rte_pci_id pci_id_cuda_map[] = {
67 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
68 NVIDIA_GPU_A40_DEVICE_ID)
71 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
72 NVIDIA_GPU_A30_24GB_DEVICE_ID)
75 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
76 NVIDIA_GPU_A30X_24GB_DPU_DEVICE_ID)
79 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
80 NVIDIA_GPU_A10_24GB_DEVICE_ID)
83 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
84 NVIDIA_GPU_A10G_DEVICE_ID)
87 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
88 NVIDIA_GPU_A10M_DEVICE_ID)
91 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
92 NVIDIA_GPU_A100_40GB_SXM4_DEVICE_ID)
95 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
96 NVIDIA_GPU_A100_40GB_PCIE_DEVICE_ID)
99 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
100 NVIDIA_GPU_A100_80GB_SXM4_DEVICE_ID)
103 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
104 NVIDIA_GPU_A100_80GB_PCIE_DEVICE_ID)
107 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
108 NVIDIA_GPU_A100X_80GB_DPU_DEVICE_ID)
111 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
112 NVIDIA_GPU_GA100_PG506_207)
115 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
116 NVIDIA_GPU_GA100_PCIE)
119 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
120 NVIDIA_GPU_GA100_PG506_217)
123 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
124 NVIDIA_GPU_V100_16GB_SXM2_DEVICE_ID)
127 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
128 NVIDIA_GPU_V100_16GB_DGXS_DEVICE_ID)
131 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
132 NVIDIA_GPU_V100_16GB_FHHL_DEVICE_ID)
135 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
136 NVIDIA_GPU_V100_16GB_PCIE_DEVICE_ID)
139 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
140 NVIDIA_GPU_V100_32GB_SXM2_DEVICE_ID)
143 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
144 NVIDIA_GPU_V100_32GB_PCIE_DEVICE_ID)
147 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
148 NVIDIA_GPU_V100_32GB_DGXS_DEVICE_ID)
151 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
152 NVIDIA_GPU_V100_32GB_SXM3_DEVICE_ID)
155 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
156 NVIDIA_GPU_V100_32GB_SXM3_H_DEVICE_ID)
159 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
160 NVIDIA_GPU_V100_SXM2)
163 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
164 NVIDIA_GPU_V100S_PCIE)
167 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
168 NVIDIA_GPU_TITAN_V_CEO_ED)
171 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
172 NVIDIA_GPU_GV100GL_PG500_216)
175 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
176 NVIDIA_GPU_GV100GL_PG503_216)
179 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
180 NVIDIA_GPU_TU102_TITAN_RTX)
183 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
184 NVIDIA_GPU_TU102GL_QUADRO_RTX)
187 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
188 NVIDIA_GPU_GV100_QUADRO_DEVICE_ID)
191 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
192 NVIDIA_GPU_QUADRO_RTX_4000)
195 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
196 NVIDIA_GPU_QUADRO_RTX_5000)
199 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
200 NVIDIA_GPU_QUADRO_RTX_6000)
203 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
204 NVIDIA_GPU_QUADRO_RTX_8000)
207 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
208 NVIDIA_GPU_QUADRO_RTX_A4000)
211 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
212 NVIDIA_GPU_QUADRO_RTX_A6000)
215 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
216 NVIDIA_GPU_QUADRO_RTX_A5000)
219 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
220 NVIDIA_GPU_QUADRO_RTX_A4500)
223 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
224 NVIDIA_GPU_QUADRO_RTX_A5500)
227 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
228 NVIDIA_GPU_QUADRO_RTX_A2000)
231 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
232 NVIDIA_GPU_QUADRO_RTX_A2000_12GB)
235 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
239 RTE_PCI_DEVICE(NVIDIA_GPU_VENDOR_ID,
247 /* Device private info */
249 char gpu_name[RTE_DEV_NAME_MAX_LEN];
252 int gdr_write_ordering;
256 /* Type of memory allocated by CUDA driver */
263 /* key associated to a memory address */
264 typedef uintptr_t cuda_ptr_key;
266 /* Single entry of the memory list */
269 CUdeviceptr ptr_orig_d;
278 struct mem_entry *prev;
279 struct mem_entry *next;
282 static struct mem_entry *mem_alloc_list_head;
283 static struct mem_entry *mem_alloc_list_tail;
284 static uint32_t mem_alloc_list_last_elem;
286 /* Load the CUDA symbols */
291 char cuda_path[1024];
293 if (getenv("CUDA_PATH_L") == NULL)
294 snprintf(cuda_path, 1024, "%s", "libcuda.so");
296 snprintf(cuda_path, 1024, "%s/%s", getenv("CUDA_PATH_L"), "libcuda.so");
298 cudalib = dlopen(cuda_path, RTLD_LAZY);
299 if (cudalib == NULL) {
300 rte_cuda_log(ERR, "Failed to find CUDA library in %s (CUDA_PATH_L=%s)",
301 cuda_path, getenv("CUDA_PATH_L"));
309 cuda_sym_func_loader(void)
314 sym_cuInit = dlsym(cudalib, "cuInit");
315 if (sym_cuInit == NULL) {
316 rte_cuda_log(ERR, "Failed to load CUDA missing symbol cuInit");
320 sym_cuDriverGetVersion = dlsym(cudalib, "cuDriverGetVersion");
321 if (sym_cuDriverGetVersion == NULL) {
322 rte_cuda_log(ERR, "Failed to load CUDA missing symbol cuDriverGetVersion");
326 sym_cuGetProcAddress = dlsym(cudalib, "cuGetProcAddress");
327 if (sym_cuGetProcAddress == NULL) {
328 rte_cuda_log(ERR, "Failed to load CUDA missing symbol cuGetProcAddress");
336 cuda_pfn_func_loader(void)
340 res = sym_cuGetProcAddress("cuGetErrorString",
341 (void **) (&pfn_cuGetErrorString), cuda_driver_version, 0);
343 rte_cuda_log(ERR, "Retrieve pfn_cuGetErrorString failed with %d", res);
347 res = sym_cuGetProcAddress("cuGetErrorName",
348 (void **)(&pfn_cuGetErrorName), cuda_driver_version, 0);
350 rte_cuda_log(ERR, "Retrieve pfn_cuGetErrorName failed with %d", res);
354 res = sym_cuGetProcAddress("cuPointerSetAttribute",
355 (void **)(&pfn_cuPointerSetAttribute), cuda_driver_version, 0);
357 rte_cuda_log(ERR, "Retrieve pfn_cuPointerSetAttribute failed with %d", res);
361 res = sym_cuGetProcAddress("cuDeviceGetAttribute",
362 (void **)(&pfn_cuDeviceGetAttribute), cuda_driver_version, 0);
364 rte_cuda_log(ERR, "Retrieve pfn_cuDeviceGetAttribute failed with %d", res);
368 res = sym_cuGetProcAddress("cuDeviceGetByPCIBusId",
369 (void **)(&pfn_cuDeviceGetByPCIBusId), cuda_driver_version, 0);
371 rte_cuda_log(ERR, "Retrieve pfn_cuDeviceGetByPCIBusId failed with %d", res);
375 res = sym_cuGetProcAddress("cuDeviceGetName",
376 (void **)(&pfn_cuDeviceGetName), cuda_driver_version, 0);
378 rte_cuda_log(ERR, "Retrieve pfn_cuDeviceGetName failed with %d", res);
382 res = sym_cuGetProcAddress("cuDevicePrimaryCtxRetain",
383 (void **)(&pfn_cuDevicePrimaryCtxRetain), cuda_driver_version, 0);
385 rte_cuda_log(ERR, "Retrieve pfn_cuDevicePrimaryCtxRetain failed with %d", res);
389 res = sym_cuGetProcAddress("cuDevicePrimaryCtxRelease",
390 (void **)(&pfn_cuDevicePrimaryCtxRelease), cuda_driver_version, 0);
392 rte_cuda_log(ERR, "Retrieve pfn_cuDevicePrimaryCtxRelease failed with %d", res);
396 res = sym_cuGetProcAddress("cuDeviceTotalMem",
397 (void **)(&pfn_cuDeviceTotalMem), cuda_driver_version, 0);
399 rte_cuda_log(ERR, "Retrieve pfn_cuDeviceTotalMem failed with %d", res);
403 res = sym_cuGetProcAddress("cuCtxGetApiVersion",
404 (void **)(&pfn_cuCtxGetApiVersion), cuda_driver_version, 0);
406 rte_cuda_log(ERR, "Retrieve pfn_cuCtxGetApiVersion failed with %d", res);
410 res = sym_cuGetProcAddress("cuCtxGetDevice",
411 (void **)(&pfn_cuCtxGetDevice), cuda_driver_version, 0);
413 rte_cuda_log(ERR, "Retrieve pfn_cuCtxGetDevice failed with %d", res);
417 res = sym_cuGetProcAddress("cuCtxSetCurrent",
418 (void **)(&pfn_cuCtxSetCurrent), cuda_driver_version, 0);
420 rte_cuda_log(ERR, "Retrieve pfn_cuCtxSetCurrent failed with %d", res);
424 res = sym_cuGetProcAddress("cuCtxGetCurrent",
425 (void **)(&pfn_cuCtxGetCurrent), cuda_driver_version, 0);
427 rte_cuda_log(ERR, "Retrieve pfn_cuCtxGetCurrent failed with %d", res);
431 res = sym_cuGetProcAddress("cuCtxGetExecAffinity",
432 (void **)(&pfn_cuCtxGetExecAffinity), cuda_driver_version, 0);
434 rte_cuda_log(ERR, "Retrieve pfn_cuCtxGetExecAffinity failed with %d", res);
438 res = sym_cuGetProcAddress("cuMemAlloc",
439 (void **)(&pfn_cuMemAlloc), cuda_driver_version, 0);
441 rte_cuda_log(ERR, "Retrieve pfn_cuMemAlloc failed with %d", res);
445 res = sym_cuGetProcAddress("cuMemFree",
446 (void **)(&pfn_cuMemFree), cuda_driver_version, 0);
448 rte_cuda_log(ERR, "Retrieve pfn_cuMemFree failed with %d", res);
452 res = sym_cuGetProcAddress("cuMemHostRegister",
453 (void **)(&pfn_cuMemHostRegister), cuda_driver_version, 0);
455 rte_cuda_log(ERR, "Retrieve pfn_cuMemHostRegister failed with %d", res);
459 res = sym_cuGetProcAddress("cuMemHostUnregister",
460 (void **)(&pfn_cuMemHostUnregister), cuda_driver_version, 0);
462 rte_cuda_log(ERR, "Retrieve pfn_cuMemHostUnregister failed with %d", res);
466 res = sym_cuGetProcAddress("cuMemHostGetDevicePointer",
467 (void **)(&pfn_cuMemHostGetDevicePointer), cuda_driver_version, 0);
469 rte_cuda_log(ERR, "Retrieve pfn_cuMemHostGetDevicePointer failed with %d", res);
473 res = sym_cuGetProcAddress("cuFlushGPUDirectRDMAWrites",
474 (void **)(&pfn_cuFlushGPUDirectRDMAWrites), cuda_driver_version, 0);
476 rte_cuda_log(ERR, "Retrieve cuFlushGPUDirectRDMAWrites failed with %d", res);
483 /* Generate a key from a memory pointer */
485 get_hash_from_ptr(void *ptr)
487 return (uintptr_t)ptr;
491 mem_list_count_item(void)
493 return mem_alloc_list_last_elem;
496 /* Initiate list of memory allocations if not done yet */
497 static struct mem_entry *
498 mem_list_add_item(void)
500 /* Initiate list of memory allocations if not done yet */
501 if (mem_alloc_list_head == NULL) {
502 mem_alloc_list_head = rte_zmalloc(NULL,
503 sizeof(struct mem_entry),
504 RTE_CACHE_LINE_SIZE);
505 if (mem_alloc_list_head == NULL) {
506 rte_cuda_log(ERR, "Failed to allocate memory for memory list");
510 mem_alloc_list_head->next = NULL;
511 mem_alloc_list_head->prev = NULL;
512 mem_alloc_list_tail = mem_alloc_list_head;
514 struct mem_entry *mem_alloc_list_cur = rte_zmalloc(NULL,
515 sizeof(struct mem_entry),
516 RTE_CACHE_LINE_SIZE);
518 if (mem_alloc_list_cur == NULL) {
519 rte_cuda_log(ERR, "Failed to allocate memory for memory list");
523 mem_alloc_list_tail->next = mem_alloc_list_cur;
524 mem_alloc_list_cur->prev = mem_alloc_list_tail;
525 mem_alloc_list_tail = mem_alloc_list_tail->next;
526 mem_alloc_list_tail->next = NULL;
529 mem_alloc_list_last_elem++;
531 return mem_alloc_list_tail;
534 static struct mem_entry *
535 mem_list_find_item(cuda_ptr_key pk)
537 struct mem_entry *mem_alloc_list_cur = NULL;
539 if (mem_alloc_list_head == NULL) {
540 rte_cuda_log(ERR, "Memory list doesn't exist");
544 if (mem_list_count_item() == 0) {
545 rte_cuda_log(ERR, "No items in memory list");
549 mem_alloc_list_cur = mem_alloc_list_head;
551 while (mem_alloc_list_cur != NULL) {
552 if (mem_alloc_list_cur->pkey == pk)
553 return mem_alloc_list_cur;
554 mem_alloc_list_cur = mem_alloc_list_cur->next;
557 return mem_alloc_list_cur;
561 mem_list_del_item(cuda_ptr_key pk)
563 struct mem_entry *mem_alloc_list_cur = NULL;
565 mem_alloc_list_cur = mem_list_find_item(pk);
566 if (mem_alloc_list_cur == NULL)
569 /* if key is in head */
570 if (mem_alloc_list_cur->prev == NULL) {
571 mem_alloc_list_head = mem_alloc_list_cur->next;
572 if (mem_alloc_list_head != NULL)
573 mem_alloc_list_head->prev = NULL;
575 mem_alloc_list_cur->prev->next = mem_alloc_list_cur->next;
576 if (mem_alloc_list_cur->next != NULL)
577 mem_alloc_list_cur->next->prev = mem_alloc_list_cur->prev;
580 rte_free(mem_alloc_list_cur);
582 mem_alloc_list_last_elem--;
588 cuda_dev_info_get(struct rte_gpu *dev, struct rte_gpu_info *info)
592 struct rte_gpu_info parent_info;
593 CUexecAffinityParam affinityPrm;
594 const char *err_string;
595 struct cuda_info *private;
596 CUcontext current_ctx;
604 /* Child initialization time probably called by rte_gpu_add_child() */
605 if (dev->mpshared->info.parent != RTE_GPU_ID_NONE &&
606 dev->mpshared->dev_private == NULL) {
607 /* Store current ctx */
608 res = pfn_cuCtxGetCurrent(¤t_ctx);
610 pfn_cuGetErrorString(res, &(err_string));
611 rte_cuda_log(ERR, "cuCtxGetCurrent failed with %s",
617 /* Set child ctx as current ctx */
618 input_ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
619 res = pfn_cuCtxSetCurrent(input_ctx);
621 pfn_cuGetErrorString(res, &(err_string));
622 rte_cuda_log(ERR, "cuCtxSetCurrent input failed with %s",
633 res = pfn_cuCtxGetExecAffinity(&affinityPrm,
634 CU_EXEC_AFFINITY_TYPE_SM_COUNT);
636 pfn_cuGetErrorString(res, &(err_string));
637 rte_cuda_log(ERR, "cuCtxGetExecAffinity failed with %s",
640 dev->mpshared->info.processor_count =
641 (uint32_t)affinityPrm.param.smCount.val;
643 ret = rte_gpu_info_get(dev->mpshared->info.parent, &parent_info);
648 dev->mpshared->info.total_memory = parent_info.total_memory;
650 dev->mpshared->info.page_size = parent_info.page_size;
653 * GPU Device private info
655 dev->mpshared->dev_private = rte_zmalloc(NULL,
656 sizeof(struct cuda_info),
657 RTE_CACHE_LINE_SIZE);
658 if (dev->mpshared->dev_private == NULL) {
659 rte_cuda_log(ERR, "Failed to allocate memory for GPU process private");
664 private = (struct cuda_info *)dev->mpshared->dev_private;
666 res = pfn_cuCtxGetDevice(&(private->cu_dev));
668 pfn_cuGetErrorString(res, &(err_string));
669 rte_cuda_log(ERR, "cuCtxGetDevice failed with %s",
675 res = pfn_cuDeviceGetName(private->gpu_name,
676 RTE_DEV_NAME_MAX_LEN, private->cu_dev);
678 pfn_cuGetErrorString(res, &(err_string));
679 rte_cuda_log(ERR, "cuDeviceGetName failed with %s",
685 /* Restore original ctx as current ctx */
686 res = pfn_cuCtxSetCurrent(current_ctx);
688 pfn_cuGetErrorString(res, &(err_string));
689 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
696 *info = dev->mpshared->info;
706 cuda_mem_alloc(struct rte_gpu *dev, size_t size, unsigned int align, void **ptr)
709 const char *err_string;
710 CUcontext current_ctx;
712 unsigned int flag = 1;
717 /* Store current ctx */
718 res = pfn_cuCtxGetCurrent(¤t_ctx);
720 pfn_cuGetErrorString(res, &(err_string));
721 rte_cuda_log(ERR, "cuCtxGetCurrent failed with %s",
727 /* Set child ctx as current ctx */
728 input_ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
729 res = pfn_cuCtxSetCurrent(input_ctx);
731 pfn_cuGetErrorString(res, &(err_string));
732 rte_cuda_log(ERR, "cuCtxSetCurrent input failed with %s",
738 /* Get next memory list item */
739 mem_alloc_list_tail = mem_list_add_item();
740 if (mem_alloc_list_tail == NULL) {
745 /* Allocate memory */
746 mem_alloc_list_tail->size = size;
747 mem_alloc_list_tail->size_orig = size + align;
749 res = pfn_cuMemAlloc(&(mem_alloc_list_tail->ptr_orig_d),
750 mem_alloc_list_tail->size_orig);
752 pfn_cuGetErrorString(res, &(err_string));
753 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
759 /* Align memory address */
760 mem_alloc_list_tail->ptr_d = mem_alloc_list_tail->ptr_orig_d;
761 if (align && ((uintptr_t)mem_alloc_list_tail->ptr_d) % align)
762 mem_alloc_list_tail->ptr_d += (align -
763 (((uintptr_t)mem_alloc_list_tail->ptr_d) % align));
765 /* GPUDirect RDMA attribute required */
766 res = pfn_cuPointerSetAttribute(&flag,
767 CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
768 mem_alloc_list_tail->ptr_d);
770 rte_cuda_log(ERR, "Could not set SYNC MEMOP attribute for "
771 "GPU memory at %"PRIu32", err %d",
772 (uint32_t)mem_alloc_list_tail->ptr_d, res);
777 mem_alloc_list_tail->pkey = get_hash_from_ptr((void *)mem_alloc_list_tail->ptr_d);
778 mem_alloc_list_tail->ptr_h = NULL;
779 mem_alloc_list_tail->dev = dev;
780 mem_alloc_list_tail->ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
781 mem_alloc_list_tail->mtype = GPU_MEM;
783 /* Restore original ctx as current ctx */
784 res = pfn_cuCtxSetCurrent(current_ctx);
786 pfn_cuGetErrorString(res, &(err_string));
787 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
793 *ptr = (void *)mem_alloc_list_tail->ptr_d;
799 cuda_mem_register(struct rte_gpu *dev, size_t size, void *ptr)
802 const char *err_string;
803 CUcontext current_ctx;
805 unsigned int flag = 1;
811 /* Store current ctx */
812 res = pfn_cuCtxGetCurrent(¤t_ctx);
814 pfn_cuGetErrorString(res, &(err_string));
815 rte_cuda_log(ERR, "cuCtxGetCurrent failed with %s",
821 /* Set child ctx as current ctx */
822 input_ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
823 res = pfn_cuCtxSetCurrent(input_ctx);
825 pfn_cuGetErrorString(res, &(err_string));
826 rte_cuda_log(ERR, "cuCtxSetCurrent input failed with %s",
832 /* Get next memory list item */
833 mem_alloc_list_tail = mem_list_add_item();
834 if (mem_alloc_list_tail == NULL) {
839 /* Allocate memory */
840 mem_alloc_list_tail->size = size;
841 mem_alloc_list_tail->ptr_h = ptr;
843 res = pfn_cuMemHostRegister(mem_alloc_list_tail->ptr_h,
844 mem_alloc_list_tail->size,
845 CU_MEMHOSTREGISTER_PORTABLE |
846 CU_MEMHOSTREGISTER_DEVICEMAP);
848 pfn_cuGetErrorString(res, &(err_string));
849 rte_cuda_log(ERR, "cuMemHostRegister failed with %s ptr %p size %zd",
851 mem_alloc_list_tail->ptr_h,
852 mem_alloc_list_tail->size);
857 res = pfn_cuDeviceGetAttribute(&(use_ptr_h),
858 CU_DEVICE_ATTRIBUTE_CAN_USE_HOST_POINTER_FOR_REGISTERED_MEM,
859 ((struct cuda_info *)(dev->mpshared->dev_private))->cu_dev);
861 pfn_cuGetErrorString(res, &(err_string));
862 rte_cuda_log(ERR, "cuDeviceGetAttribute failed with %s",
868 if (use_ptr_h == 0) {
869 res = pfn_cuMemHostGetDevicePointer(&(mem_alloc_list_tail->ptr_d),
870 mem_alloc_list_tail->ptr_h, 0);
872 pfn_cuGetErrorString(res, &(err_string));
873 rte_cuda_log(ERR, "cuMemHostGetDevicePointer failed with %s",
879 if ((uintptr_t)mem_alloc_list_tail->ptr_d !=
880 (uintptr_t)mem_alloc_list_tail->ptr_h) {
881 rte_cuda_log(ERR, "Host input pointer is different wrt GPU registered pointer");
886 mem_alloc_list_tail->ptr_d = (CUdeviceptr)mem_alloc_list_tail->ptr_h;
889 /* GPUDirect RDMA attribute required */
890 res = pfn_cuPointerSetAttribute(&flag,
891 CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
892 mem_alloc_list_tail->ptr_d);
894 rte_cuda_log(ERR, "Could not set SYNC MEMOP attribute for GPU memory at %"PRIu32
895 ", err %d", (uint32_t)mem_alloc_list_tail->ptr_d, res);
900 mem_alloc_list_tail->pkey = get_hash_from_ptr((void *)mem_alloc_list_tail->ptr_h);
901 mem_alloc_list_tail->size = size;
902 mem_alloc_list_tail->dev = dev;
903 mem_alloc_list_tail->ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
904 mem_alloc_list_tail->mtype = CPU_REGISTERED;
905 mem_alloc_list_tail->ptr_orig_d = mem_alloc_list_tail->ptr_d;
907 /* Restore original ctx as current ctx */
908 res = pfn_cuCtxSetCurrent(current_ctx);
910 pfn_cuGetErrorString(res, &(err_string));
911 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
921 cuda_mem_cpu_map(struct rte_gpu *dev, __rte_unused size_t size, void *ptr_in, void **ptr_out)
923 struct mem_entry *mem_item;
929 hk = get_hash_from_ptr((void *)ptr_in);
931 mem_item = mem_list_find_item(hk);
932 if (mem_item == NULL) {
933 rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory.", ptr_in);
938 if (mem_item->mtype != GPU_MEM) {
939 rte_cuda_log(ERR, "Memory address 0x%p is not GPU memory type.", ptr_in);
944 if (mem_item->size != size)
945 rte_cuda_log(WARNING,
946 "Can't expose memory area with size (%zd) different from original size (%zd).",
947 size, mem_item->size);
949 if (gdrcopy_pin(&gdrc_h, &(mem_item->mh), (uint64_t)mem_item->ptr_d,
950 mem_item->size, &(mem_item->ptr_h))) {
951 rte_cuda_log(ERR, "Error exposing GPU memory address 0x%p.", ptr_in);
956 mem_item->mtype = GPU_REGISTERED;
957 *ptr_out = mem_item->ptr_h;
963 cuda_mem_unregister(struct rte_gpu *dev, void *ptr)
966 struct mem_entry *mem_item;
967 const char *err_string;
973 hk = get_hash_from_ptr((void *)ptr);
975 mem_item = mem_list_find_item(hk);
976 if (mem_item == NULL) {
977 rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory", ptr);
982 if (mem_item->mtype == CPU_REGISTERED) {
983 res = pfn_cuMemHostUnregister(ptr);
985 pfn_cuGetErrorString(res, &(err_string));
986 rte_cuda_log(ERR, "cuMemHostUnregister current failed with %s",
992 return mem_list_del_item(hk);
995 rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
1002 cuda_mem_cpu_unmap(struct rte_gpu *dev, void *ptr_in)
1004 struct mem_entry *mem_item;
1010 hk = get_hash_from_ptr((void *)ptr_in);
1012 mem_item = mem_list_find_item(hk);
1013 if (mem_item == NULL) {
1014 rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory.", ptr_in);
1019 if (mem_item->mtype == GPU_REGISTERED) {
1020 if (gdrcopy_unpin(gdrc_h, mem_item->mh, (void *)mem_item->ptr_d,
1022 rte_cuda_log(ERR, "Error unexposing GPU memory address 0x%p.", ptr_in);
1027 mem_item->mtype = GPU_MEM;
1037 cuda_mem_free(struct rte_gpu *dev, void *ptr)
1040 struct mem_entry *mem_item;
1041 const char *err_string;
1047 hk = get_hash_from_ptr((void *)ptr);
1049 mem_item = mem_list_find_item(hk);
1050 if (mem_item == NULL) {
1051 rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory", ptr);
1057 * If a GPU memory area that's CPU mapped is being freed
1058 * without calling cpu_unmap, force the unmapping.
1060 if (mem_item->mtype == GPU_REGISTERED)
1061 cuda_mem_cpu_unmap(dev, ptr);
1063 if (mem_item->mtype == GPU_MEM) {
1064 res = pfn_cuMemFree(mem_item->ptr_orig_d);
1066 pfn_cuGetErrorString(res, &(err_string));
1067 rte_cuda_log(ERR, "cuMemFree current failed with %s",
1073 return mem_list_del_item(hk);
1076 rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
1082 cuda_dev_close(struct rte_gpu *dev)
1087 rte_free(dev->mpshared->dev_private);
1093 cuda_wmb(struct rte_gpu *dev)
1096 const char *err_string;
1097 CUcontext current_ctx;
1098 CUcontext input_ctx;
1099 struct cuda_info *private;
1106 private = (struct cuda_info *)dev->mpshared->dev_private;
1108 if (private->gdr_write_ordering != CU_GPU_DIRECT_RDMA_WRITES_ORDERING_NONE) {
1110 * No need to explicitly force the write ordering because
1111 * the device natively supports it
1116 if (private->gdr_flush_type != CU_FLUSH_GPU_DIRECT_RDMA_WRITES_OPTION_HOST) {
1118 * Can't flush GDR writes with cuFlushGPUDirectRDMAWrites CUDA function.
1119 * Application needs to use alternative methods.
1121 rte_cuda_log(WARNING, "Can't flush GDR writes with cuFlushGPUDirectRDMAWrites CUDA function."
1122 "Application needs to use alternative methods.");
1124 rte_errno = ENOTSUP;
1128 /* Store current ctx */
1129 res = pfn_cuCtxGetCurrent(¤t_ctx);
1131 pfn_cuGetErrorString(res, &(err_string));
1132 rte_cuda_log(ERR, "cuCtxGetCurrent failed with %s",
1138 /* Set child ctx as current ctx */
1139 input_ctx = (CUcontext)((uintptr_t)dev->mpshared->info.context);
1140 res = pfn_cuCtxSetCurrent(input_ctx);
1142 pfn_cuGetErrorString(res, &(err_string));
1143 rte_cuda_log(ERR, "cuCtxSetCurrent input failed with %s",
1149 res = pfn_cuFlushGPUDirectRDMAWrites(CU_FLUSH_GPU_DIRECT_RDMA_WRITES_TARGET_CURRENT_CTX,
1150 CU_FLUSH_GPU_DIRECT_RDMA_WRITES_TO_ALL_DEVICES);
1152 pfn_cuGetErrorString(res, &(err_string));
1153 rte_cuda_log(ERR, "cuFlushGPUDirectRDMAWrites current failed with %s",
1159 /* Restore original ctx as current ctx */
1160 res = pfn_cuCtxSetCurrent(current_ctx);
1162 pfn_cuGetErrorString(res, &(err_string));
1163 rte_cuda_log(ERR, "cuCtxSetCurrent current failed with %s",
1173 cuda_gpu_probe(__rte_unused struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
1175 struct rte_gpu *dev = NULL;
1179 char dev_name[RTE_DEV_NAME_MAX_LEN];
1180 const char *err_string;
1181 int processor_count = 0;
1182 struct cuda_info *private;
1184 if (pci_dev == NULL) {
1185 rte_cuda_log(ERR, "NULL PCI device");
1190 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
1192 /* Allocate memory to be used privately by drivers */
1193 dev = rte_gpu_allocate(pci_dev->device.name);
1199 /* Initialize values only for the first CUDA driver call */
1200 if (dev->mpshared->info.dev_id == 0) {
1201 mem_alloc_list_head = NULL;
1202 mem_alloc_list_tail = NULL;
1203 mem_alloc_list_last_elem = 0;
1205 /* Load libcuda.so library */
1206 if (cuda_loader()) {
1207 rte_cuda_log(ERR, "CUDA Driver library not found");
1208 rte_errno = ENOTSUP;
1212 /* Load initial CUDA functions */
1213 if (cuda_sym_func_loader()) {
1214 rte_cuda_log(ERR, "CUDA functions not found in library");
1215 rte_errno = ENOTSUP;
1220 * Required to initialize the CUDA Driver.
1221 * Multiple calls of cuInit() will return immediately
1222 * without making any relevant change
1226 res = sym_cuDriverGetVersion(&cuda_driver_version);
1228 rte_cuda_log(ERR, "cuDriverGetVersion failed with %d", res);
1229 rte_errno = ENOTSUP;
1233 if (cuda_driver_version < CUDA_DRIVER_MIN_VERSION) {
1234 rte_cuda_log(ERR, "CUDA Driver version found is %d. "
1235 "Minimum requirement is %d",
1236 cuda_driver_version,
1237 CUDA_DRIVER_MIN_VERSION);
1238 rte_errno = ENOTSUP;
1242 if (cuda_pfn_func_loader()) {
1243 rte_cuda_log(ERR, "CUDA PFN functions not found in library");
1244 rte_errno = ENOTSUP;
1251 /* Fill HW specific part of device structure */
1252 dev->device = &pci_dev->device;
1253 dev->mpshared->info.numa_node = pci_dev->device.numa_node;
1255 /* Get NVIDIA GPU Device descriptor */
1256 res = pfn_cuDeviceGetByPCIBusId(&cu_dev_id, dev->device->name);
1258 pfn_cuGetErrorString(res, &(err_string));
1259 rte_cuda_log(ERR, "cuDeviceGetByPCIBusId name %s failed with %d: %s",
1260 dev->device->name, res, err_string);
1265 res = pfn_cuDevicePrimaryCtxRetain(&pctx, cu_dev_id);
1267 pfn_cuGetErrorString(res, &(err_string));
1268 rte_cuda_log(ERR, "cuDevicePrimaryCtxRetain name %s failed with %d: %s",
1269 dev->device->name, res, err_string);
1274 res = pfn_cuCtxGetApiVersion(pctx, &cuda_api_version);
1276 rte_cuda_log(ERR, "cuCtxGetApiVersion failed with %d", res);
1277 rte_errno = ENOTSUP;
1281 if (cuda_api_version < CUDA_API_MIN_VERSION) {
1282 rte_cuda_log(ERR, "CUDA API version found is %d Minimum requirement is %d",
1283 cuda_api_version, CUDA_API_MIN_VERSION);
1284 rte_errno = ENOTSUP;
1288 dev->mpshared->info.context = (uint64_t)pctx;
1291 * GPU Device generic info
1294 /* Processor count */
1295 res = pfn_cuDeviceGetAttribute(&(processor_count),
1296 CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT,
1299 pfn_cuGetErrorString(res, &(err_string));
1300 rte_cuda_log(ERR, "cuDeviceGetAttribute failed with %s",
1305 dev->mpshared->info.processor_count = (uint32_t)processor_count;
1308 res = pfn_cuDeviceTotalMem(&dev->mpshared->info.total_memory, cu_dev_id);
1310 pfn_cuGetErrorString(res, &(err_string));
1311 rte_cuda_log(ERR, "cuDeviceTotalMem failed with %s",
1317 dev->mpshared->info.page_size = (size_t)GPU_PAGE_SIZE;
1320 * GPU Device private info
1322 dev->mpshared->dev_private = rte_zmalloc(NULL,
1323 sizeof(struct cuda_info),
1324 RTE_CACHE_LINE_SIZE);
1325 if (dev->mpshared->dev_private == NULL) {
1326 rte_cuda_log(ERR, "Failed to allocate memory for GPU process private");
1331 private = (struct cuda_info *)dev->mpshared->dev_private;
1332 private->cu_dev = cu_dev_id;
1333 res = pfn_cuDeviceGetName(private->gpu_name,
1334 RTE_DEV_NAME_MAX_LEN,
1337 pfn_cuGetErrorString(res, &(err_string));
1338 rte_cuda_log(ERR, "cuDeviceGetName failed with %s",
1344 res = pfn_cuDeviceGetAttribute(&(private->gdr_supported),
1345 CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_SUPPORTED,
1348 pfn_cuGetErrorString(res, &(err_string));
1349 rte_cuda_log(ERR, "cuDeviceGetAttribute failed with %s",
1355 if (private->gdr_supported == 0)
1356 rte_cuda_log(WARNING, "GPU %s doesn't support GPUDirect RDMA",
1357 pci_dev->device.name);
1359 res = pfn_cuDeviceGetAttribute(&(private->gdr_write_ordering),
1360 CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WRITES_ORDERING,
1363 pfn_cuGetErrorString(res, &(err_string));
1365 "cuDeviceGetAttribute failed with %s",
1371 if (private->gdr_write_ordering == CU_GPU_DIRECT_RDMA_WRITES_ORDERING_NONE) {
1372 res = pfn_cuDeviceGetAttribute(&(private->gdr_flush_type),
1373 CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_FLUSH_WRITES_OPTIONS,
1376 pfn_cuGetErrorString(res, &(err_string));
1377 rte_cuda_log(ERR, "cuDeviceGetAttribute failed with %s",
1383 if (private->gdr_flush_type != CU_FLUSH_GPU_DIRECT_RDMA_WRITES_OPTION_HOST)
1384 rte_cuda_log(ERR, "GPUDirect RDMA flush writes API is not supported");
1387 dev->ops.dev_info_get = cuda_dev_info_get;
1388 dev->ops.dev_close = cuda_dev_close;
1389 dev->ops.mem_alloc = cuda_mem_alloc;
1390 dev->ops.mem_free = cuda_mem_free;
1391 dev->ops.mem_register = cuda_mem_register;
1392 dev->ops.mem_unregister = cuda_mem_unregister;
1393 dev->ops.mem_cpu_map = cuda_mem_cpu_map;
1394 dev->ops.mem_cpu_unmap = cuda_mem_cpu_unmap;
1395 dev->ops.wmb = cuda_wmb;
1397 rte_gpu_complete_new(dev);
1399 rte_cuda_debug("dev id = %u name = %s",
1400 dev->mpshared->info.dev_id, private->gpu_name);
1406 cuda_gpu_remove(struct rte_pci_device *pci_dev)
1408 struct rte_gpu *dev;
1412 if (pci_dev == NULL) {
1417 dev = rte_gpu_get_by_name(pci_dev->device.name);
1419 rte_cuda_log(ERR, "Couldn't find HW dev \"%s\" to uninitialise it",
1420 pci_dev->device.name);
1424 gpu_id = dev->mpshared->info.dev_id;
1426 /* release dev from library */
1427 ret = rte_gpu_release(dev);
1429 rte_cuda_log(ERR, "Device %i failed to uninit: %i", gpu_id, ret);
1431 rte_cuda_debug("Destroyed dev = %u", gpu_id);
1436 static struct rte_pci_driver rte_cuda_driver = {
1437 .id_table = pci_id_cuda_map,
1438 .drv_flags = RTE_PCI_DRV_WC_ACTIVATE,
1439 .probe = cuda_gpu_probe,
1440 .remove = cuda_gpu_remove,
1443 RTE_PMD_REGISTER_PCI(gpu_cuda, rte_cuda_driver);
1444 RTE_PMD_REGISTER_PCI_TABLE(gpu_cuda, pci_id_cuda_map);
1445 RTE_PMD_REGISTER_KMOD_DEP(gpu_cuda, "* nvidia & (nv_peer_mem | nvpeer_mem)");