gpu/cuda: unmap GPU memory while freeing

Enable GPU_REGISTERED flag in gpu/cuda driver in the memory list.
If a GPU memory address CPU mapped is freed before being
unmapped, CUDA driver unmaps it before freeing the memory.

Signed-off-by: Elena Agostini <eagostini@nvidia.com>

diff --git a/drivers/gpu/cuda/cuda.c b/drivers/gpu/cuda/cuda.c
index 5b8476ac20d9e99a929f0c369c04c922983b643f..664605d9fbbee08b50892066d2b783dc913c9dcf 100644 (file)
--- a/drivers/gpu/cuda/cuda.c
+++ b/drivers/gpu/cuda/cuda.c
@@ -257,7 +257,7 @@ struct cuda_info {
 enum mem_type {
        GPU_MEM = 0,
        CPU_REGISTERED,
-       GPU_REGISTERED /* Not used yet */
+       GPU_REGISTERED
 };
 
 /* key associated to a memory address */
@@ -953,13 +953,14 @@ cuda_mem_cpu_map(struct rte_gpu *dev, __rte_unused size_t size, void *ptr_in, vo
                return -rte_errno;
        }
 
+       mem_item->mtype = GPU_REGISTERED;
        *ptr_out = mem_item->ptr_h;
 
        return 0;
 }
 
 static int
-cuda_mem_free(struct rte_gpu *dev, void *ptr)
+cuda_mem_unregister(struct rte_gpu *dev, void *ptr)
 {
        CUresult res;
        struct mem_entry *mem_item;
@@ -978,11 +979,11 @@ cuda_mem_free(struct rte_gpu *dev, void *ptr)
                return -rte_errno;
        }
 
-       if (mem_item->mtype == GPU_MEM) {
-               res = pfn_cuMemFree(mem_item->ptr_orig_d);
+       if (mem_item->mtype == CPU_REGISTERED) {
+               res = pfn_cuMemHostUnregister(ptr);
                if (res != 0) {
                        pfn_cuGetErrorString(res, &(err_string));
-                       rte_cuda_log(ERR, "cuMemFree current failed with %s",
+                       rte_cuda_log(ERR, "cuMemHostUnregister current failed with %s",
                                        err_string);
                        rte_errno = EPERM;
                        return -rte_errno;
@@ -993,74 +994,88 @@ cuda_mem_free(struct rte_gpu *dev, void *ptr)
 
        rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
 
-       return -EPERM;
+       rte_errno = EPERM;
+       return -rte_errno;
 }
 
 static int
-cuda_mem_unregister(struct rte_gpu *dev, void *ptr)
+cuda_mem_cpu_unmap(struct rte_gpu *dev, void *ptr_in)
 {
-       CUresult res;
        struct mem_entry *mem_item;
-       const char *err_string;
        cuda_ptr_key hk;
 
        if (dev == NULL)
                return -ENODEV;
 
-       hk = get_hash_from_ptr((void *)ptr);
+       hk = get_hash_from_ptr((void *)ptr_in);
 
        mem_item = mem_list_find_item(hk);
        if (mem_item == NULL) {
-               rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory", ptr);
+               rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory.", ptr_in);
                rte_errno = EPERM;
                return -rte_errno;
        }
 
-       if (mem_item->mtype == CPU_REGISTERED) {
-               res = pfn_cuMemHostUnregister(ptr);
-               if (res != 0) {
-                       pfn_cuGetErrorString(res, &(err_string));
-                       rte_cuda_log(ERR, "cuMemHostUnregister current failed with %s",
-                                       err_string);
+       if (mem_item->mtype == GPU_REGISTERED) {
+               if (gdrcopy_unpin(gdrc_h, mem_item->mh, (void *)mem_item->ptr_d,
+                               mem_item->size)) {
+                       rte_cuda_log(ERR, "Error unexposing GPU memory address 0x%p.", ptr_in);
                        rte_errno = EPERM;
                        return -rte_errno;
                }
 
-               return mem_list_del_item(hk);
+               mem_item->mtype = GPU_MEM;
+       } else {
+               rte_errno = EPERM;
+               return -rte_errno;
        }
 
-       rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
-
-       rte_errno = EPERM;
-       return -rte_errno;
+       return 0;
 }
 
 static int
-cuda_mem_cpu_unmap(struct rte_gpu *dev, void *ptr_in)
+cuda_mem_free(struct rte_gpu *dev, void *ptr)
 {
+       CUresult res;
        struct mem_entry *mem_item;
+       const char *err_string;
        cuda_ptr_key hk;
 
        if (dev == NULL)
                return -ENODEV;
 
-       hk = get_hash_from_ptr((void *)ptr_in);
+       hk = get_hash_from_ptr((void *)ptr);
 
        mem_item = mem_list_find_item(hk);
        if (mem_item == NULL) {
-               rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory.", ptr_in);
+               rte_cuda_log(ERR, "Memory address 0x%p not found in driver memory", ptr);
                rte_errno = EPERM;
                return -rte_errno;
        }
 
-       if (gdrcopy_unpin(gdrc_h, mem_item->mh, (void *)mem_item->ptr_d,
-                       mem_item->size)) {
-               rte_cuda_log(ERR, "Error unexposing GPU memory address 0x%p.", ptr_in);
-               rte_errno = EPERM;
-               return -rte_errno;
+       /*
+        * If a GPU memory area that's CPU mapped is being freed
+        * without calling cpu_unmap, force the unmapping.
+        */
+       if (mem_item->mtype == GPU_REGISTERED)
+               cuda_mem_cpu_unmap(dev, ptr);
+
+       if (mem_item->mtype == GPU_MEM) {
+               res = pfn_cuMemFree(mem_item->ptr_orig_d);
+               if (res != 0) {
+                       pfn_cuGetErrorString(res, &(err_string));
+                       rte_cuda_log(ERR, "cuMemFree current failed with %s",
+                                       err_string);
+                       rte_errno = EPERM;
+                       return -rte_errno;
+               }
+
+               return mem_list_del_item(hk);
        }
 
-       return 0;
+       rte_cuda_log(ERR, "Memory type %d not supported", mem_item->mtype);
+
+       return -EPERM;
 }
 
 static int