X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Fgpudev%2Fgpudev.c;h=ce92d632573e40c3ab337a3fb3e4e78b77cefde3;hb=6e858b4d9244cf53505589673755ab18ac2a4a83;hp=aeb021f2ccfb3ad27f0fdd6f7d8c3e61d5b268b4;hpb=8b8036a66e3d59ffa58afb8d96fa2c73262155a7;p=dpdk.git diff --git a/lib/gpudev/gpudev.c b/lib/gpudev/gpudev.c index aeb021f2cc..ce92d63257 100644 --- a/lib/gpudev/gpudev.c +++ b/lib/gpudev/gpudev.c @@ -3,7 +3,11 @@ */ #include +#include +#include #include +#include +#include #include #include @@ -27,6 +31,22 @@ static int16_t gpu_max; /* Number of currently valid devices */ static int16_t gpu_count; +/* Shared memory between processes. */ +static const char *GPU_MEMZONE = "rte_gpu_shared"; +static struct { + __extension__ struct rte_gpu_mpshared gpus[0]; +} *gpu_shared_mem; + +/* Event callback object */ +struct rte_gpu_callback { + TAILQ_ENTRY(rte_gpu_callback) next; + rte_gpu_callback_t *function; + void *user_data; + enum rte_gpu_event event; +}; +static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER; +static void gpu_free_callbacks(struct rte_gpu *dev); + int rte_gpu_init(size_t dev_max) { @@ -64,18 +84,27 @@ bool rte_gpu_is_valid(int16_t dev_id) { if (dev_id >= 0 && dev_id < gpu_max && - gpus[dev_id].state == RTE_GPU_STATE_INITIALIZED) + gpus[dev_id].process_state == RTE_GPU_STATE_INITIALIZED) return true; return false; } +static bool +gpu_match_parent(int16_t dev_id, int16_t parent) +{ + if (parent == RTE_GPU_ID_ANY) + return true; + return gpus[dev_id].mpshared->info.parent == parent; +} + int16_t -rte_gpu_find_next(int16_t dev_id) +rte_gpu_find_next(int16_t dev_id, int16_t parent) { if (dev_id < 0) dev_id = 0; while (dev_id < gpu_max && - gpus[dev_id].state == RTE_GPU_STATE_UNUSED) + (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED || + !gpu_match_parent(dev_id, parent))) dev_id++; if (dev_id >= gpu_max) @@ -89,7 +118,7 @@ gpu_find_free_id(void) int16_t dev_id; for (dev_id = 0; dev_id < gpu_max; dev_id++) { - if (gpus[dev_id].state == RTE_GPU_STATE_UNUSED) + if (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED) return dev_id; } return RTE_GPU_ID_NONE; @@ -116,12 +145,35 @@ rte_gpu_get_by_name(const char *name) RTE_GPU_FOREACH(dev_id) { dev = &gpus[dev_id]; - if (strncmp(name, dev->name, RTE_DEV_NAME_MAX_LEN) == 0) + if (strncmp(name, dev->mpshared->name, RTE_DEV_NAME_MAX_LEN) == 0) return dev; } return NULL; } +static int +gpu_shared_mem_init(void) +{ + const struct rte_memzone *memzone; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + memzone = rte_memzone_reserve(GPU_MEMZONE, + sizeof(*gpu_shared_mem) + + sizeof(*gpu_shared_mem->gpus) * gpu_max, + SOCKET_ID_ANY, 0); + } else { + memzone = rte_memzone_lookup(GPU_MEMZONE); + } + if (memzone == NULL) { + GPU_LOG(ERR, "cannot initialize shared memory"); + rte_errno = ENOMEM; + return -rte_errno; + } + + gpu_shared_mem = memzone->addr; + return 0; +} + struct rte_gpu * rte_gpu_allocate(const char *name) { @@ -143,6 +195,10 @@ rte_gpu_allocate(const char *name) if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) return NULL; + /* initialize shared memory before adding first device */ + if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) + return NULL; + if (rte_gpu_get_by_name(name) != NULL) { GPU_LOG(ERR, "device with name %s already exists", name); rte_errno = EEXIST; @@ -158,14 +214,20 @@ rte_gpu_allocate(const char *name) dev = &gpus[dev_id]; memset(dev, 0, sizeof(*dev)); - if (rte_strscpy(dev->name, name, RTE_DEV_NAME_MAX_LEN) < 0) { + dev->mpshared = &gpu_shared_mem->gpus[dev_id]; + memset(dev->mpshared, 0, sizeof(*dev->mpshared)); + + if (rte_strscpy(dev->mpshared->name, name, RTE_DEV_NAME_MAX_LEN) < 0) { GPU_LOG(ERR, "device name too long: %s", name); rte_errno = ENAMETOOLONG; return NULL; } - dev->info.name = dev->name; - dev->info.dev_id = dev_id; - dev->info.numa_node = -1; + dev->mpshared->info.name = dev->mpshared->name; + dev->mpshared->info.dev_id = dev_id; + dev->mpshared->info.numa_node = -1; + dev->mpshared->info.parent = RTE_GPU_ID_NONE; + TAILQ_INIT(&dev->callbacks); + __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); gpu_count++; GPU_LOG(DEBUG, "new device %s (id %d) of total %d", @@ -173,26 +235,111 @@ rte_gpu_allocate(const char *name) return dev; } +struct rte_gpu * +rte_gpu_attach(const char *name) +{ + int16_t dev_id; + struct rte_gpu *dev; + struct rte_gpu_mpshared *shared_dev; + + if (rte_eal_process_type() != RTE_PROC_SECONDARY) { + GPU_LOG(ERR, "only secondary process can attach device"); + rte_errno = EPERM; + return NULL; + } + if (name == NULL) { + GPU_LOG(ERR, "attach device without a name"); + rte_errno = EINVAL; + return NULL; + } + + /* implicit initialization of library before adding first device */ + if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0) + return NULL; + + /* initialize shared memory before adding first device */ + if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0) + return NULL; + + for (dev_id = 0; dev_id < gpu_max; dev_id++) { + shared_dev = &gpu_shared_mem->gpus[dev_id]; + if (strncmp(name, shared_dev->name, RTE_DEV_NAME_MAX_LEN) == 0) + break; + } + if (dev_id >= gpu_max) { + GPU_LOG(ERR, "device with name %s not found", name); + rte_errno = ENOENT; + return NULL; + } + dev = &gpus[dev_id]; + memset(dev, 0, sizeof(*dev)); + + TAILQ_INIT(&dev->callbacks); + dev->mpshared = shared_dev; + __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); + + gpu_count++; + GPU_LOG(DEBUG, "attached device %s (id %d) of total %d", + name, dev_id, gpu_count); + return dev; +} + +int16_t +rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context) +{ + struct rte_gpu *dev; + + if (!rte_gpu_is_valid(parent)) { + GPU_LOG(ERR, "add child to invalid parent ID %d", parent); + rte_errno = ENODEV; + return -rte_errno; + } + + dev = rte_gpu_allocate(name); + if (dev == NULL) + return -rte_errno; + + dev->mpshared->info.parent = parent; + dev->mpshared->info.context = child_context; + + rte_gpu_complete_new(dev); + return dev->mpshared->info.dev_id; +} + void rte_gpu_complete_new(struct rte_gpu *dev) { if (dev == NULL) return; - dev->state = RTE_GPU_STATE_INITIALIZED; + dev->process_state = RTE_GPU_STATE_INITIALIZED; + rte_gpu_notify(dev, RTE_GPU_EVENT_NEW); } int rte_gpu_release(struct rte_gpu *dev) { + int16_t dev_id, child; + if (dev == NULL) { rte_errno = ENODEV; return -rte_errno; } + dev_id = dev->mpshared->info.dev_id; + RTE_GPU_FOREACH_CHILD(child, dev_id) { + GPU_LOG(ERR, "cannot release device %d with child %d", + dev_id, child); + rte_errno = EBUSY; + return -rte_errno; + } GPU_LOG(DEBUG, "free device %s (id %d)", - dev->info.name, dev->info.dev_id); - dev->state = RTE_GPU_STATE_UNUSED; + dev->mpshared->info.name, dev->mpshared->info.dev_id); + rte_gpu_notify(dev, RTE_GPU_EVENT_DEL); + + gpu_free_callbacks(dev); + dev->process_state = RTE_GPU_STATE_UNUSED; + __atomic_fetch_sub(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED); gpu_count--; return 0; @@ -224,6 +371,137 @@ rte_gpu_close(int16_t dev_id) return firsterr; } +int +rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event, + rte_gpu_callback_t *function, void *user_data) +{ + int16_t next_dev, last_dev; + struct rte_gpu_callback_list *callbacks; + struct rte_gpu_callback *callback; + + if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { + GPU_LOG(ERR, "register callback of invalid ID %d", dev_id); + rte_errno = ENODEV; + return -rte_errno; + } + if (function == NULL) { + GPU_LOG(ERR, "cannot register callback without function"); + rte_errno = EINVAL; + return -rte_errno; + } + + if (dev_id == RTE_GPU_ID_ANY) { + next_dev = 0; + last_dev = gpu_max - 1; + } else { + next_dev = last_dev = dev_id; + } + + rte_rwlock_write_lock(&gpu_callback_lock); + do { + callbacks = &gpus[next_dev].callbacks; + + /* check if not already registered */ + TAILQ_FOREACH(callback, callbacks, next) { + if (callback->event == event && + callback->function == function && + callback->user_data == user_data) { + GPU_LOG(INFO, "callback already registered"); + return 0; + } + } + + callback = malloc(sizeof(*callback)); + if (callback == NULL) { + GPU_LOG(ERR, "cannot allocate callback"); + return -ENOMEM; + } + callback->function = function; + callback->user_data = user_data; + callback->event = event; + TAILQ_INSERT_TAIL(callbacks, callback, next); + + } while (++next_dev <= last_dev); + rte_rwlock_write_unlock(&gpu_callback_lock); + + return 0; +} + +int +rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event, + rte_gpu_callback_t *function, void *user_data) +{ + int16_t next_dev, last_dev; + struct rte_gpu_callback_list *callbacks; + struct rte_gpu_callback *callback, *nextcb; + + if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) { + GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id); + rte_errno = ENODEV; + return -rte_errno; + } + if (function == NULL) { + GPU_LOG(ERR, "cannot unregister callback without function"); + rte_errno = EINVAL; + return -rte_errno; + } + + if (dev_id == RTE_GPU_ID_ANY) { + next_dev = 0; + last_dev = gpu_max - 1; + } else { + next_dev = last_dev = dev_id; + } + + rte_rwlock_write_lock(&gpu_callback_lock); + do { + callbacks = &gpus[next_dev].callbacks; + RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { + if (callback->event != event || + callback->function != function || + (callback->user_data != user_data && + user_data != (void *)-1)) + continue; + TAILQ_REMOVE(callbacks, callback, next); + free(callback); + } + } while (++next_dev <= last_dev); + rte_rwlock_write_unlock(&gpu_callback_lock); + + return 0; +} + +static void +gpu_free_callbacks(struct rte_gpu *dev) +{ + struct rte_gpu_callback_list *callbacks; + struct rte_gpu_callback *callback, *nextcb; + + callbacks = &dev->callbacks; + rte_rwlock_write_lock(&gpu_callback_lock); + RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) { + TAILQ_REMOVE(callbacks, callback, next); + free(callback); + } + rte_rwlock_write_unlock(&gpu_callback_lock); +} + +void +rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event) +{ + int16_t dev_id; + struct rte_gpu_callback *callback; + + dev_id = dev->mpshared->info.dev_id; + rte_rwlock_read_lock(&gpu_callback_lock); + TAILQ_FOREACH(callback, &dev->callbacks, next) { + if (callback->event != event || callback->function == NULL) + continue; + callback->function(dev_id, event, callback->user_data); + } + rte_rwlock_read_unlock(&gpu_callback_lock); +} + int rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info) { @@ -242,8 +520,463 @@ rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info) } if (dev->ops.dev_info_get == NULL) { - *info = dev->info; + *info = dev->mpshared->info; return 0; } return GPU_DRV_RET(dev->ops.dev_info_get(dev, info)); } + +void * +rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align) +{ + struct rte_gpu *dev; + void *ptr; + int ret; + + dev = gpu_get_by_id(dev_id); + if (dev == NULL) { + GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); + rte_errno = ENODEV; + return NULL; + } + + if (dev->ops.mem_alloc == NULL) { + GPU_LOG(ERR, "mem allocation not supported"); + rte_errno = ENOTSUP; + return NULL; + } + + if (size == 0) /* dry-run */ + return NULL; + + if (align && !rte_is_power_of_2(align)) { + GPU_LOG(ERR, "requested alignment is not a power of two %u", align); + rte_errno = EINVAL; + return NULL; + } + + ret = dev->ops.mem_alloc(dev, size, align, &ptr); + + switch (ret) { + case 0: + return ptr; + case -ENOMEM: + case -E2BIG: + rte_errno = -ret; + return NULL; + default: + rte_errno = -EPERM; + return NULL; + } +} + +int +rte_gpu_mem_free(int16_t dev_id, void *ptr) +{ + struct rte_gpu *dev; + + dev = gpu_get_by_id(dev_id); + if (dev == NULL) { + GPU_LOG(ERR, "free mem for invalid device ID %d", dev_id); + rte_errno = ENODEV; + return -rte_errno; + } + + if (dev->ops.mem_free == NULL) { + rte_errno = ENOTSUP; + return -rte_errno; + } + + if (ptr == NULL) /* dry-run */ + return 0; + + return GPU_DRV_RET(dev->ops.mem_free(dev, ptr)); +} + +int +rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr) +{ + struct rte_gpu *dev; + + dev = gpu_get_by_id(dev_id); + if (dev == NULL) { + GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id); + rte_errno = ENODEV; + return -rte_errno; + } + + if (dev->ops.mem_register == NULL) { + GPU_LOG(ERR, "mem registration not supported"); + rte_errno = ENOTSUP; + return -rte_errno; + } + + if (ptr == NULL || size == 0) /* dry-run */ + return 0; + + return GPU_DRV_RET(dev->ops.mem_register(dev, size, ptr)); +} + +int +rte_gpu_mem_unregister(int16_t dev_id, void *ptr) +{ + struct rte_gpu *dev; + + dev = gpu_get_by_id(dev_id); + if (dev == NULL) { + GPU_LOG(ERR, "unregister mem for invalid device ID %d", dev_id); + rte_errno = ENODEV; + return -rte_errno; + } + + if (dev->ops.mem_unregister == NULL) { + rte_errno = ENOTSUP; + return -rte_errno; + } + + if (ptr == NULL) /* dry-run */ + return 0; + + return GPU_DRV_RET(dev->ops.mem_unregister(dev, ptr)); +} + +void * +rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr) +{ + struct rte_gpu *dev; + void *ptr_out; + int ret; + + dev = gpu_get_by_id(dev_id); + if (dev == NULL) { + GPU_LOG(ERR, "mem CPU map for invalid device ID %d", dev_id); + rte_errno = ENODEV; + return NULL; + } + + if (dev->ops.mem_cpu_map == NULL) { + GPU_LOG(ERR, "mem CPU map not supported"); + rte_errno = ENOTSUP; + return NULL; + } + + if (ptr == NULL || size == 0) /* dry-run */ + return NULL; + + ret = GPU_DRV_RET(dev->ops.mem_cpu_map(dev, size, ptr, &ptr_out)); + + switch (ret) { + case 0: + return ptr_out; + case -ENOMEM: + case -E2BIG: + rte_errno = -ret; + return NULL; + default: + rte_errno = -EPERM; + return NULL; + } +} + +int +rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr) +{ + struct rte_gpu *dev; + + dev = gpu_get_by_id(dev_id); + if (dev == NULL) { + GPU_LOG(ERR, "cpu_unmap mem for invalid device ID %d", dev_id); + rte_errno = ENODEV; + return -rte_errno; + } + + if (dev->ops.mem_cpu_unmap == NULL) { + rte_errno = ENOTSUP; + return -rte_errno; + } + + if (ptr == NULL) /* dry-run */ + return 0; + + return GPU_DRV_RET(dev->ops.mem_cpu_unmap(dev, ptr)); +} + +int +rte_gpu_wmb(int16_t dev_id) +{ + struct rte_gpu *dev; + + dev = gpu_get_by_id(dev_id); + if (dev == NULL) { + GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); + rte_errno = ENODEV; + return -rte_errno; + } + + if (dev->ops.wmb == NULL) { + rte_errno = ENOTSUP; + return -rte_errno; + } + return GPU_DRV_RET(dev->ops.wmb(dev)); +} + +int +rte_gpu_comm_create_flag(uint16_t dev_id, struct rte_gpu_comm_flag *devflag, + enum rte_gpu_comm_flag_type mtype) +{ + size_t flag_size; + int ret; + + if (devflag == NULL) { + rte_errno = EINVAL; + return -rte_errno; + } + if (mtype != RTE_GPU_COMM_FLAG_CPU) { + rte_errno = EINVAL; + return -rte_errno; + } + + flag_size = sizeof(uint32_t); + + devflag->ptr = rte_zmalloc(NULL, flag_size, 0); + if (devflag->ptr == NULL) { + rte_errno = ENOMEM; + return -rte_errno; + } + + ret = rte_gpu_mem_register(dev_id, flag_size, devflag->ptr); + if (ret < 0) { + rte_errno = ENOMEM; + return -rte_errno; + } + + devflag->mtype = mtype; + devflag->dev_id = dev_id; + + return 0; +} + +int +rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag) +{ + int ret; + + if (devflag == NULL) { + rte_errno = EINVAL; + return -rte_errno; + } + + ret = rte_gpu_mem_unregister(devflag->dev_id, devflag->ptr); + if (ret < 0) { + rte_errno = EINVAL; + return -1; + } + + rte_free(devflag->ptr); + + return 0; +} + +int +rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, uint32_t val) +{ + if (devflag == NULL) { + rte_errno = EINVAL; + return -rte_errno; + } + + if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { + rte_errno = EINVAL; + return -rte_errno; + } + + RTE_GPU_VOLATILE(*devflag->ptr) = val; + + return 0; +} + +int +rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, uint32_t *val) +{ + if (devflag == NULL) { + rte_errno = EINVAL; + return -rte_errno; + } + if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) { + rte_errno = EINVAL; + return -rte_errno; + } + + *val = RTE_GPU_VOLATILE(*devflag->ptr); + + return 0; +} + +struct rte_gpu_comm_list * +rte_gpu_comm_create_list(uint16_t dev_id, + uint32_t num_comm_items) +{ + struct rte_gpu_comm_list *comm_list; + uint32_t idx_l; + int ret; + struct rte_gpu *dev; + + if (num_comm_items == 0) { + rte_errno = EINVAL; + return NULL; + } + + dev = gpu_get_by_id(dev_id); + if (dev == NULL) { + GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id); + rte_errno = ENODEV; + return NULL; + } + + comm_list = rte_zmalloc(NULL, + sizeof(struct rte_gpu_comm_list) * num_comm_items, 0); + if (comm_list == NULL) { + rte_errno = ENOMEM; + return NULL; + } + + ret = rte_gpu_mem_register(dev_id, + sizeof(struct rte_gpu_comm_list) * num_comm_items, comm_list); + if (ret < 0) { + rte_errno = ENOMEM; + return NULL; + } + + for (idx_l = 0; idx_l < num_comm_items; idx_l++) { + comm_list[idx_l].pkt_list = rte_zmalloc(NULL, + sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); + if (comm_list[idx_l].pkt_list == NULL) { + rte_errno = ENOMEM; + return NULL; + } + + ret = rte_gpu_mem_register(dev_id, + sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, + comm_list[idx_l].pkt_list); + if (ret < 0) { + rte_errno = ENOMEM; + return NULL; + } + + RTE_GPU_VOLATILE(comm_list[idx_l].status) = RTE_GPU_COMM_LIST_FREE; + comm_list[idx_l].num_pkts = 0; + comm_list[idx_l].dev_id = dev_id; + + comm_list[idx_l].mbufs = rte_zmalloc(NULL, + sizeof(struct rte_mbuf *) * RTE_GPU_COMM_LIST_PKTS_MAX, 0); + if (comm_list[idx_l].mbufs == NULL) { + rte_errno = ENOMEM; + return NULL; + } + } + + return comm_list; +} + +int +rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list, + uint32_t num_comm_items) +{ + uint32_t idx_l; + int ret; + uint16_t dev_id; + + if (comm_list == NULL) { + rte_errno = EINVAL; + return -rte_errno; + } + + dev_id = comm_list[0].dev_id; + + for (idx_l = 0; idx_l < num_comm_items; idx_l++) { + ret = rte_gpu_mem_unregister(dev_id, comm_list[idx_l].pkt_list); + if (ret < 0) { + rte_errno = EINVAL; + return -1; + } + + rte_free(comm_list[idx_l].pkt_list); + rte_free(comm_list[idx_l].mbufs); + } + + ret = rte_gpu_mem_unregister(dev_id, comm_list); + if (ret < 0) { + rte_errno = EINVAL; + return -1; + } + + rte_free(comm_list); + + return 0; +} + +int +rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item, + struct rte_mbuf **mbufs, uint32_t num_mbufs) +{ + uint32_t idx; + + if (comm_list_item == NULL || comm_list_item->pkt_list == NULL || + mbufs == NULL || num_mbufs > RTE_GPU_COMM_LIST_PKTS_MAX) { + rte_errno = EINVAL; + return -rte_errno; + } + + for (idx = 0; idx < num_mbufs; idx++) { + /* support only unchained mbufs */ + if (unlikely((mbufs[idx]->nb_segs > 1) || + (mbufs[idx]->next != NULL) || + (mbufs[idx]->data_len != mbufs[idx]->pkt_len))) { + rte_errno = ENOTSUP; + return -rte_errno; + } + comm_list_item->pkt_list[idx].addr = + rte_pktmbuf_mtod_offset(mbufs[idx], uintptr_t, 0); + comm_list_item->pkt_list[idx].size = mbufs[idx]->pkt_len; + comm_list_item->mbufs[idx] = mbufs[idx]; + } + + RTE_GPU_VOLATILE(comm_list_item->num_pkts) = num_mbufs; + rte_gpu_wmb(comm_list_item->dev_id); + RTE_GPU_VOLATILE(comm_list_item->status) = RTE_GPU_COMM_LIST_READY; + + return 0; +} + +int +rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item) +{ + uint32_t idx = 0; + + if (comm_list_item == NULL) { + rte_errno = EINVAL; + return -rte_errno; + } + + if (RTE_GPU_VOLATILE(comm_list_item->status) == + RTE_GPU_COMM_LIST_READY) { + GPU_LOG(ERR, "packet list is still in progress"); + rte_errno = EINVAL; + return -rte_errno; + } + + for (idx = 0; idx < RTE_GPU_COMM_LIST_PKTS_MAX; idx++) { + if (comm_list_item->pkt_list[idx].addr == 0) + break; + + comm_list_item->pkt_list[idx].addr = 0; + comm_list_item->pkt_list[idx].size = 0; + comm_list_item->mbufs[idx] = NULL; + } + + RTE_GPU_VOLATILE(comm_list_item->status) = RTE_GPU_COMM_LIST_FREE; + RTE_GPU_VOLATILE(comm_list_item->num_pkts) = 0; + rte_mb(); + + return 0; +}