1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
9 #include <rte_common.h>
10 #include <rte_errno.h>
12 #include <rte_debug.h>
14 #include <rte_malloc.h>
15 #include <rte_mempool.h>
16 #include <rte_memzone.h>
17 #include <rte_lcore.h>
19 #include <rte_spinlock.h>
20 #include <rte_tailq.h>
21 #include <rte_interrupts.h>
23 #include "rte_bbdev_op.h"
24 #include "rte_bbdev.h"
25 #include "rte_bbdev_pmd.h"
27 #define DEV_NAME "BBDEV"
30 /* Helper macro to check dev_id is valid */
31 #define VALID_DEV_OR_RET_ERR(dev, dev_id) do { \
33 rte_bbdev_log(ERR, "device %u is invalid", dev_id); \
38 /* Helper macro to check dev_ops is valid */
39 #define VALID_DEV_OPS_OR_RET_ERR(dev, dev_id) do { \
40 if (dev->dev_ops == NULL) { \
41 rte_bbdev_log(ERR, "NULL dev_ops structure in device %u", \
47 /* Helper macro to check that driver implements required function pointer */
48 #define VALID_FUNC_OR_RET_ERR(func, dev_id) do { \
50 rte_bbdev_log(ERR, "device %u does not support %s", \
56 /* Helper macro to check that queue is valid */
57 #define VALID_QUEUE_OR_RET_ERR(queue_id, dev) do { \
58 if (queue_id >= dev->data->num_queues) { \
59 rte_bbdev_log(ERR, "Invalid queue_id %u for device %u", \
60 queue_id, dev->data->dev_id); \
65 /* List of callback functions registered by an application */
66 struct rte_bbdev_callback {
67 TAILQ_ENTRY(rte_bbdev_callback) next; /* Callbacks list */
68 rte_bbdev_cb_fn cb_fn; /* Callback address */
69 void *cb_arg; /* Parameter for callback */
70 void *ret_param; /* Return parameter */
71 enum rte_bbdev_event_type event; /* Interrupt event type */
72 uint32_t active; /* Callback is executing */
75 /* spinlock for bbdev device callbacks */
76 static rte_spinlock_t rte_bbdev_cb_lock = RTE_SPINLOCK_INITIALIZER;
79 * Global array of all devices. This is not static because it's used by the
80 * inline enqueue and dequeue functions
82 struct rte_bbdev rte_bbdev_devices[RTE_BBDEV_MAX_DEVS];
84 /* Global array with rte_bbdev_data structures */
85 static struct rte_bbdev_data *rte_bbdev_data;
87 /* Memzone name for global bbdev data pool */
88 static const char *MZ_RTE_BBDEV_DATA = "rte_bbdev_data";
90 /* Number of currently valid devices */
91 static uint16_t num_devs;
93 /* Return pointer to device structure, with validity check */
94 static struct rte_bbdev *
95 get_dev(uint16_t dev_id)
97 if (rte_bbdev_is_valid(dev_id))
98 return &rte_bbdev_devices[dev_id];
102 /* Allocate global data array */
104 rte_bbdev_data_alloc(void)
106 const unsigned int flags = 0;
107 const struct rte_memzone *mz;
109 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
110 mz = rte_memzone_reserve(MZ_RTE_BBDEV_DATA,
111 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data),
112 rte_socket_id(), flags);
114 mz = rte_memzone_lookup(MZ_RTE_BBDEV_DATA);
117 "Cannot allocate memzone for bbdev port data");
121 rte_bbdev_data = mz->addr;
122 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
123 memset(rte_bbdev_data, 0,
124 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data));
129 * Find data alocated for the device or if not found return first unused bbdev
130 * data. If all structures are in use and none is used by the device return
133 static struct rte_bbdev_data *
134 find_bbdev_data(const char *name)
138 for (data_id = 0; data_id < RTE_BBDEV_MAX_DEVS; ++data_id) {
139 if (strlen(rte_bbdev_data[data_id].name) == 0) {
140 memset(&rte_bbdev_data[data_id], 0,
141 sizeof(struct rte_bbdev_data));
142 return &rte_bbdev_data[data_id];
143 } else if (strncmp(rte_bbdev_data[data_id].name, name,
144 RTE_BBDEV_NAME_MAX_LEN) == 0)
145 return &rte_bbdev_data[data_id];
151 /* Find lowest device id with no attached device */
153 find_free_dev_id(void)
156 for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
157 if (rte_bbdev_devices[i].state == RTE_BBDEV_UNUSED)
160 return RTE_BBDEV_MAX_DEVS;
164 rte_bbdev_allocate(const char *name)
167 struct rte_bbdev *bbdev;
171 rte_bbdev_log(ERR, "Invalid null device name");
175 if (rte_bbdev_get_named_dev(name) != NULL) {
176 rte_bbdev_log(ERR, "Device \"%s\" is already allocated", name);
180 dev_id = find_free_dev_id();
181 if (dev_id == RTE_BBDEV_MAX_DEVS) {
182 rte_bbdev_log(ERR, "Reached maximum number of devices");
186 bbdev = &rte_bbdev_devices[dev_id];
188 if (rte_bbdev_data == NULL) {
189 ret = rte_bbdev_data_alloc();
194 bbdev->data = find_bbdev_data(name);
195 if (bbdev->data == NULL) {
197 "Max BBDevs already allocated in multi-process environment!");
201 rte_atomic16_inc(&bbdev->data->process_cnt);
202 bbdev->data->dev_id = dev_id;
203 bbdev->state = RTE_BBDEV_INITIALIZED;
205 ret = snprintf(bbdev->data->name, RTE_BBDEV_NAME_MAX_LEN, "%s", name);
206 if ((ret < 0) || (ret >= RTE_BBDEV_NAME_MAX_LEN)) {
207 rte_bbdev_log(ERR, "Copying device name \"%s\" failed", name);
211 /* init user callbacks */
212 TAILQ_INIT(&(bbdev->list_cbs));
216 rte_bbdev_log_debug("Initialised device %s (id = %u). Num devices = %u",
217 name, dev_id, num_devs);
223 rte_bbdev_release(struct rte_bbdev *bbdev)
226 struct rte_bbdev_callback *cb, *next;
229 rte_bbdev_log(ERR, "NULL bbdev");
232 dev_id = bbdev->data->dev_id;
234 /* free all callbacks from the device's list */
235 for (cb = TAILQ_FIRST(&bbdev->list_cbs); cb != NULL; cb = next) {
237 next = TAILQ_NEXT(cb, next);
238 TAILQ_REMOVE(&(bbdev->list_cbs), cb, next);
242 /* clear shared BBDev Data if no process is using the device anymore */
243 if (rte_atomic16_dec_and_test(&bbdev->data->process_cnt))
244 memset(bbdev->data, 0, sizeof(*bbdev->data));
246 memset(bbdev, 0, sizeof(*bbdev));
248 bbdev->state = RTE_BBDEV_UNUSED;
251 "Un-initialised device id = %u. Num devices = %u",
257 rte_bbdev_get_named_dev(const char *name)
262 rte_bbdev_log(ERR, "NULL driver name");
266 for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
267 struct rte_bbdev *dev = get_dev(i);
268 if (dev && (strncmp(dev->data->name,
269 name, RTE_BBDEV_NAME_MAX_LEN) == 0))
277 rte_bbdev_count(void)
283 rte_bbdev_is_valid(uint16_t dev_id)
285 if ((dev_id < RTE_BBDEV_MAX_DEVS) &&
286 rte_bbdev_devices[dev_id].state == RTE_BBDEV_INITIALIZED)
292 rte_bbdev_find_next(uint16_t dev_id)
295 for (; dev_id < RTE_BBDEV_MAX_DEVS; dev_id++)
296 if (rte_bbdev_is_valid(dev_id))
302 rte_bbdev_setup_queues(uint16_t dev_id, uint16_t num_queues, int socket_id)
306 struct rte_bbdev_driver_info dev_info;
307 struct rte_bbdev *dev = get_dev(dev_id);
308 VALID_DEV_OR_RET_ERR(dev, dev_id);
310 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
312 if (dev->data->started) {
314 "Device %u cannot be configured when started",
319 /* Get device driver information to get max number of queues */
320 VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
321 memset(&dev_info, 0, sizeof(dev_info));
322 dev->dev_ops->info_get(dev, &dev_info);
324 if ((num_queues == 0) || (num_queues > dev_info.max_num_queues)) {
326 "Device %u supports 0 < N <= %u queues, not %u",
327 dev_id, dev_info.max_num_queues, num_queues);
331 /* If re-configuration, get driver to free existing internal memory */
332 if (dev->data->queues != NULL) {
333 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
334 for (i = 0; i < dev->data->num_queues; i++) {
335 int ret = dev->dev_ops->queue_release(dev, i);
338 "Device %u queue %u release failed",
343 /* Call optional device close */
344 if (dev->dev_ops->close) {
345 ret = dev->dev_ops->close(dev);
348 "Device %u couldn't be closed",
353 rte_free(dev->data->queues);
356 /* Allocate queue pointers */
357 dev->data->queues = rte_calloc_socket(DEV_NAME, num_queues,
358 sizeof(dev->data->queues[0]), RTE_CACHE_LINE_SIZE,
359 dev->data->socket_id);
360 if (dev->data->queues == NULL) {
362 "calloc of %u queues for device %u on socket %i failed",
363 num_queues, dev_id, dev->data->socket_id);
367 dev->data->num_queues = num_queues;
369 /* Call optional device configuration */
370 if (dev->dev_ops->setup_queues) {
371 ret = dev->dev_ops->setup_queues(dev, num_queues, socket_id);
374 "Device %u memory configuration failed",
380 rte_bbdev_log_debug("Device %u set up with %u queues", dev_id,
385 dev->data->num_queues = 0;
386 rte_free(dev->data->queues);
387 dev->data->queues = NULL;
392 rte_bbdev_intr_enable(uint16_t dev_id)
395 struct rte_bbdev *dev = get_dev(dev_id);
396 VALID_DEV_OR_RET_ERR(dev, dev_id);
398 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
400 if (dev->data->started) {
402 "Device %u cannot be configured when started",
407 if (dev->dev_ops->intr_enable) {
408 ret = dev->dev_ops->intr_enable(dev);
411 "Device %u interrupts configuration failed",
415 rte_bbdev_log_debug("Enabled interrupts for dev %u", dev_id);
419 rte_bbdev_log(ERR, "Device %u doesn't support interrupts", dev_id);
424 rte_bbdev_queue_configure(uint16_t dev_id, uint16_t queue_id,
425 const struct rte_bbdev_queue_conf *conf)
428 struct rte_bbdev_driver_info dev_info;
429 struct rte_bbdev *dev = get_dev(dev_id);
430 const struct rte_bbdev_op_cap *p;
431 struct rte_bbdev_queue_conf *stored_conf;
432 const char *op_type_str;
433 VALID_DEV_OR_RET_ERR(dev, dev_id);
435 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
437 VALID_QUEUE_OR_RET_ERR(queue_id, dev);
439 if (dev->data->queues[queue_id].started || dev->data->started) {
441 "Queue %u of device %u cannot be configured when started",
446 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
447 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_setup, dev_id);
449 /* Get device driver information to verify config is valid */
450 VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
451 memset(&dev_info, 0, sizeof(dev_info));
452 dev->dev_ops->info_get(dev, &dev_info);
454 /* Check configuration is valid */
456 if ((conf->op_type == RTE_BBDEV_OP_NONE) &&
457 (dev_info.capabilities[0].type ==
458 RTE_BBDEV_OP_NONE)) {
461 for (p = dev_info.capabilities;
462 p->type != RTE_BBDEV_OP_NONE; p++) {
463 if (conf->op_type == p->type) {
470 rte_bbdev_log(ERR, "Invalid operation type");
473 if (conf->queue_size > dev_info.queue_size_lim) {
475 "Size (%u) of queue %u of device %u must be: <= %u",
476 conf->queue_size, queue_id, dev_id,
477 dev_info.queue_size_lim);
480 if (!rte_is_power_of_2(conf->queue_size)) {
482 "Size (%u) of queue %u of device %u must be a power of 2",
483 conf->queue_size, queue_id, dev_id);
486 if (conf->priority > dev_info.max_queue_priority) {
488 "Priority (%u) of queue %u of bdev %u must be <= %u",
489 conf->priority, queue_id, dev_id,
490 dev_info.max_queue_priority);
495 /* Release existing queue (in case of queue reconfiguration) */
496 if (dev->data->queues[queue_id].queue_private != NULL) {
497 ret = dev->dev_ops->queue_release(dev, queue_id);
499 rte_bbdev_log(ERR, "Device %u queue %u release failed",
505 /* Get driver to setup the queue */
506 ret = dev->dev_ops->queue_setup(dev, queue_id, (conf != NULL) ?
507 conf : &dev_info.default_queue_conf);
510 "Device %u queue %u setup failed", dev_id,
515 /* Store configuration */
516 stored_conf = &dev->data->queues[queue_id].conf;
518 (conf != NULL) ? conf : &dev_info.default_queue_conf,
519 sizeof(*stored_conf));
521 op_type_str = rte_bbdev_op_type_str(stored_conf->op_type);
522 if (op_type_str == NULL)
525 rte_bbdev_log_debug("Configured dev%uq%u (size=%u, type=%s, prio=%u)",
526 dev_id, queue_id, stored_conf->queue_size, op_type_str,
527 stored_conf->priority);
533 rte_bbdev_start(uint16_t dev_id)
536 struct rte_bbdev *dev = get_dev(dev_id);
537 VALID_DEV_OR_RET_ERR(dev, dev_id);
539 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
541 if (dev->data->started) {
542 rte_bbdev_log_debug("Device %u is already started", dev_id);
546 if (dev->dev_ops->start) {
547 int ret = dev->dev_ops->start(dev);
549 rte_bbdev_log(ERR, "Device %u start failed", dev_id);
554 /* Store new state */
555 for (i = 0; i < dev->data->num_queues; i++)
556 if (!dev->data->queues[i].conf.deferred_start)
557 dev->data->queues[i].started = true;
558 dev->data->started = true;
560 rte_bbdev_log_debug("Started device %u", dev_id);
565 rte_bbdev_stop(uint16_t dev_id)
567 struct rte_bbdev *dev = get_dev(dev_id);
568 VALID_DEV_OR_RET_ERR(dev, dev_id);
570 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
572 if (!dev->data->started) {
573 rte_bbdev_log_debug("Device %u is already stopped", dev_id);
577 if (dev->dev_ops->stop)
578 dev->dev_ops->stop(dev);
579 dev->data->started = false;
581 rte_bbdev_log_debug("Stopped device %u", dev_id);
586 rte_bbdev_close(uint16_t dev_id)
590 struct rte_bbdev *dev = get_dev(dev_id);
591 VALID_DEV_OR_RET_ERR(dev, dev_id);
593 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
595 if (dev->data->started) {
596 ret = rte_bbdev_stop(dev_id);
598 rte_bbdev_log(ERR, "Device %u stop failed", dev_id);
603 /* Free memory used by queues */
604 for (i = 0; i < dev->data->num_queues; i++) {
605 ret = dev->dev_ops->queue_release(dev, i);
607 rte_bbdev_log(ERR, "Device %u queue %u release failed",
612 rte_free(dev->data->queues);
614 if (dev->dev_ops->close) {
615 ret = dev->dev_ops->close(dev);
617 rte_bbdev_log(ERR, "Device %u close failed", dev_id);
622 /* Clear configuration */
623 dev->data->queues = NULL;
624 dev->data->num_queues = 0;
626 rte_bbdev_log_debug("Closed device %u", dev_id);
631 rte_bbdev_queue_start(uint16_t dev_id, uint16_t queue_id)
633 struct rte_bbdev *dev = get_dev(dev_id);
634 VALID_DEV_OR_RET_ERR(dev, dev_id);
636 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
638 VALID_QUEUE_OR_RET_ERR(queue_id, dev);
640 if (dev->data->queues[queue_id].started) {
641 rte_bbdev_log_debug("Queue %u of device %u already started",
646 if (dev->dev_ops->queue_start) {
647 int ret = dev->dev_ops->queue_start(dev, queue_id);
649 rte_bbdev_log(ERR, "Device %u queue %u start failed",
654 dev->data->queues[queue_id].started = true;
656 rte_bbdev_log_debug("Started queue %u of device %u", queue_id, dev_id);
661 rte_bbdev_queue_stop(uint16_t dev_id, uint16_t queue_id)
663 struct rte_bbdev *dev = get_dev(dev_id);
664 VALID_DEV_OR_RET_ERR(dev, dev_id);
666 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
668 VALID_QUEUE_OR_RET_ERR(queue_id, dev);
670 if (!dev->data->queues[queue_id].started) {
671 rte_bbdev_log_debug("Queue %u of device %u already stopped",
676 if (dev->dev_ops->queue_stop) {
677 int ret = dev->dev_ops->queue_stop(dev, queue_id);
679 rte_bbdev_log(ERR, "Device %u queue %u stop failed",
684 dev->data->queues[queue_id].started = false;
686 rte_bbdev_log_debug("Stopped queue %u of device %u", queue_id, dev_id);
690 /* Get device statistics */
692 get_stats_from_queues(struct rte_bbdev *dev, struct rte_bbdev_stats *stats)
695 for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
696 struct rte_bbdev_stats *q_stats =
697 &dev->data->queues[q_id].queue_stats;
699 stats->enqueued_count += q_stats->enqueued_count;
700 stats->dequeued_count += q_stats->dequeued_count;
701 stats->enqueue_err_count += q_stats->enqueue_err_count;
702 stats->dequeue_err_count += q_stats->dequeue_err_count;
704 rte_bbdev_log_debug("Got stats on %u", dev->data->dev_id);
708 reset_stats_in_queues(struct rte_bbdev *dev)
711 for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
712 struct rte_bbdev_stats *q_stats =
713 &dev->data->queues[q_id].queue_stats;
715 memset(q_stats, 0, sizeof(*q_stats));
717 rte_bbdev_log_debug("Reset stats on %u", dev->data->dev_id);
721 rte_bbdev_stats_get(uint16_t dev_id, struct rte_bbdev_stats *stats)
723 struct rte_bbdev *dev = get_dev(dev_id);
724 VALID_DEV_OR_RET_ERR(dev, dev_id);
726 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
729 rte_bbdev_log(ERR, "NULL stats structure");
733 memset(stats, 0, sizeof(*stats));
734 if (dev->dev_ops->stats_get != NULL)
735 dev->dev_ops->stats_get(dev, stats);
737 get_stats_from_queues(dev, stats);
739 rte_bbdev_log_debug("Retrieved stats of device %u", dev_id);
744 rte_bbdev_stats_reset(uint16_t dev_id)
746 struct rte_bbdev *dev = get_dev(dev_id);
747 VALID_DEV_OR_RET_ERR(dev, dev_id);
749 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
751 if (dev->dev_ops->stats_reset != NULL)
752 dev->dev_ops->stats_reset(dev);
754 reset_stats_in_queues(dev);
756 rte_bbdev_log_debug("Reset stats of device %u", dev_id);
761 rte_bbdev_info_get(uint16_t dev_id, struct rte_bbdev_info *dev_info)
763 struct rte_bbdev *dev = get_dev(dev_id);
764 VALID_DEV_OR_RET_ERR(dev, dev_id);
766 VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
768 if (dev_info == NULL) {
769 rte_bbdev_log(ERR, "NULL dev info structure");
773 /* Copy data maintained by device interface layer */
774 memset(dev_info, 0, sizeof(*dev_info));
775 dev_info->dev_name = dev->data->name;
776 dev_info->num_queues = dev->data->num_queues;
777 dev_info->bus = rte_bus_find_by_device(dev->device);
778 dev_info->socket_id = dev->data->socket_id;
779 dev_info->started = dev->data->started;
781 /* Copy data maintained by device driver layer */
782 dev->dev_ops->info_get(dev, &dev_info->drv);
784 rte_bbdev_log_debug("Retrieved info of device %u", dev_id);
789 rte_bbdev_queue_info_get(uint16_t dev_id, uint16_t queue_id,
790 struct rte_bbdev_queue_info *queue_info)
792 struct rte_bbdev *dev = get_dev(dev_id);
793 VALID_DEV_OR_RET_ERR(dev, dev_id);
795 VALID_QUEUE_OR_RET_ERR(queue_id, dev);
797 if (queue_info == NULL) {
798 rte_bbdev_log(ERR, "NULL queue info structure");
802 /* Copy data to output */
803 memset(queue_info, 0, sizeof(*queue_info));
804 queue_info->conf = dev->data->queues[queue_id].conf;
805 queue_info->started = dev->data->queues[queue_id].started;
807 rte_bbdev_log_debug("Retrieved info of queue %u of device %u",
812 /* Calculate size needed to store bbdev_op, depending on type */
814 get_bbdev_op_size(enum rte_bbdev_op_type type)
816 unsigned int result = 0;
818 case RTE_BBDEV_OP_NONE:
819 result = RTE_MAX(sizeof(struct rte_bbdev_dec_op),
820 sizeof(struct rte_bbdev_enc_op));
822 case RTE_BBDEV_OP_TURBO_DEC:
823 result = sizeof(struct rte_bbdev_dec_op);
825 case RTE_BBDEV_OP_TURBO_ENC:
826 result = sizeof(struct rte_bbdev_enc_op);
835 /* Initialise a bbdev_op structure */
837 bbdev_op_init(struct rte_mempool *mempool, void *arg, void *element,
838 __rte_unused unsigned int n)
840 enum rte_bbdev_op_type type = *(enum rte_bbdev_op_type *)arg;
842 if (type == RTE_BBDEV_OP_TURBO_DEC) {
843 struct rte_bbdev_dec_op *op = element;
844 memset(op, 0, mempool->elt_size);
845 op->mempool = mempool;
846 } else if (type == RTE_BBDEV_OP_TURBO_ENC) {
847 struct rte_bbdev_enc_op *op = element;
848 memset(op, 0, mempool->elt_size);
849 op->mempool = mempool;
854 rte_bbdev_op_pool_create(const char *name, enum rte_bbdev_op_type type,
855 unsigned int num_elements, unsigned int cache_size,
858 struct rte_bbdev_op_pool_private *priv;
859 struct rte_mempool *mp;
860 const char *op_type_str;
863 rte_bbdev_log(ERR, "NULL name for op pool");
867 if (type >= RTE_BBDEV_OP_TYPE_COUNT) {
869 "Invalid op type (%u), should be less than %u",
870 type, RTE_BBDEV_OP_TYPE_COUNT);
874 mp = rte_mempool_create(name, num_elements, get_bbdev_op_size(type),
875 cache_size, sizeof(struct rte_bbdev_op_pool_private),
876 NULL, NULL, bbdev_op_init, &type, socket_id, 0);
879 "Failed to create op pool %s (num ops=%u, op size=%u) with error: %s",
880 name, num_elements, get_bbdev_op_size(type),
881 rte_strerror(rte_errno));
885 op_type_str = rte_bbdev_op_type_str(type);
886 if (op_type_str == NULL)
890 "Op pool %s created for %u ops (type=%s, cache=%u, socket=%u, size=%u)",
891 name, num_elements, op_type_str, cache_size, socket_id,
892 get_bbdev_op_size(type));
894 priv = (struct rte_bbdev_op_pool_private *)rte_mempool_get_priv(mp);
901 rte_bbdev_callback_register(uint16_t dev_id, enum rte_bbdev_event_type event,
902 rte_bbdev_cb_fn cb_fn, void *cb_arg)
904 struct rte_bbdev_callback *user_cb;
905 struct rte_bbdev *dev = get_dev(dev_id);
906 VALID_DEV_OR_RET_ERR(dev, dev_id);
908 if (event >= RTE_BBDEV_EVENT_MAX) {
910 "Invalid event type (%u), should be less than %u",
911 event, RTE_BBDEV_EVENT_MAX);
916 rte_bbdev_log(ERR, "NULL callback function");
920 rte_spinlock_lock(&rte_bbdev_cb_lock);
922 TAILQ_FOREACH(user_cb, &(dev->list_cbs), next) {
923 if (user_cb->cb_fn == cb_fn &&
924 user_cb->cb_arg == cb_arg &&
925 user_cb->event == event)
929 /* create a new callback. */
930 if (user_cb == NULL) {
931 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
932 sizeof(struct rte_bbdev_callback), 0);
933 if (user_cb != NULL) {
934 user_cb->cb_fn = cb_fn;
935 user_cb->cb_arg = cb_arg;
936 user_cb->event = event;
937 TAILQ_INSERT_TAIL(&(dev->list_cbs), user_cb, next);
941 rte_spinlock_unlock(&rte_bbdev_cb_lock);
942 return (user_cb == NULL) ? -ENOMEM : 0;
946 rte_bbdev_callback_unregister(uint16_t dev_id, enum rte_bbdev_event_type event,
947 rte_bbdev_cb_fn cb_fn, void *cb_arg)
950 struct rte_bbdev_callback *cb, *next;
951 struct rte_bbdev *dev = get_dev(dev_id);
952 VALID_DEV_OR_RET_ERR(dev, dev_id);
954 if (event >= RTE_BBDEV_EVENT_MAX) {
956 "Invalid event type (%u), should be less than %u",
957 event, RTE_BBDEV_EVENT_MAX);
963 "NULL callback function cannot be unregistered");
967 dev = &rte_bbdev_devices[dev_id];
968 rte_spinlock_lock(&rte_bbdev_cb_lock);
970 for (cb = TAILQ_FIRST(&dev->list_cbs); cb != NULL; cb = next) {
972 next = TAILQ_NEXT(cb, next);
974 if (cb->cb_fn != cb_fn || cb->event != event ||
975 (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
978 /* If this callback is not executing right now, remove it. */
979 if (cb->active == 0) {
980 TAILQ_REMOVE(&(dev->list_cbs), cb, next);
986 rte_spinlock_unlock(&rte_bbdev_cb_lock);
991 rte_bbdev_pmd_callback_process(struct rte_bbdev *dev,
992 enum rte_bbdev_event_type event, void *ret_param)
994 struct rte_bbdev_callback *cb_lst;
995 struct rte_bbdev_callback dev_cb;
998 rte_bbdev_log(ERR, "NULL device");
1002 if (dev->data == NULL) {
1003 rte_bbdev_log(ERR, "NULL data structure");
1007 if (event >= RTE_BBDEV_EVENT_MAX) {
1009 "Invalid event type (%u), should be less than %u",
1010 event, RTE_BBDEV_EVENT_MAX);
1014 rte_spinlock_lock(&rte_bbdev_cb_lock);
1015 TAILQ_FOREACH(cb_lst, &(dev->list_cbs), next) {
1016 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1020 if (ret_param != NULL)
1021 dev_cb.ret_param = ret_param;
1023 rte_spinlock_unlock(&rte_bbdev_cb_lock);
1024 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1025 dev_cb.cb_arg, dev_cb.ret_param);
1026 rte_spinlock_lock(&rte_bbdev_cb_lock);
1029 rte_spinlock_unlock(&rte_bbdev_cb_lock);
1033 rte_bbdev_queue_intr_enable(uint16_t dev_id, uint16_t queue_id)
1035 struct rte_bbdev *dev = get_dev(dev_id);
1036 VALID_DEV_OR_RET_ERR(dev, dev_id);
1037 VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1038 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1039 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_enable, dev_id);
1040 return dev->dev_ops->queue_intr_enable(dev, queue_id);
1044 rte_bbdev_queue_intr_disable(uint16_t dev_id, uint16_t queue_id)
1046 struct rte_bbdev *dev = get_dev(dev_id);
1047 VALID_DEV_OR_RET_ERR(dev, dev_id);
1048 VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1049 VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1050 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_disable, dev_id);
1051 return dev->dev_ops->queue_intr_disable(dev, queue_id);
1055 rte_bbdev_queue_intr_ctl(uint16_t dev_id, uint16_t queue_id, int epfd, int op,
1059 struct rte_bbdev *dev = get_dev(dev_id);
1060 struct rte_intr_handle *intr_handle;
1063 VALID_DEV_OR_RET_ERR(dev, dev_id);
1064 VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1066 intr_handle = dev->intr_handle;
1067 if (!intr_handle || !intr_handle->intr_vec) {
1068 rte_bbdev_log(ERR, "Device %u intr handle unset\n", dev_id);
1072 if (queue_id >= RTE_MAX_RXTX_INTR_VEC_ID) {
1073 rte_bbdev_log(ERR, "Device %u queue_id %u is too big\n",
1078 vec = intr_handle->intr_vec[queue_id];
1079 ret = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
1080 if (ret && (ret != -EEXIST)) {
1082 "dev %u q %u int ctl error op %d epfd %d vec %u\n",
1083 dev_id, queue_id, op, epfd, vec);
1092 rte_bbdev_op_type_str(enum rte_bbdev_op_type op_type)
1094 static const char * const op_types[] = {
1095 "RTE_BBDEV_OP_NONE",
1096 "RTE_BBDEV_OP_TURBO_DEC",
1097 "RTE_BBDEV_OP_TURBO_ENC",
1100 if (op_type < RTE_BBDEV_OP_TYPE_COUNT)
1101 return op_types[op_type];
1103 rte_bbdev_log(ERR, "Invalid operation type");
1110 RTE_INIT(rte_bbdev_init_log);
1112 rte_bbdev_init_log(void)
1114 bbdev_logtype = rte_log_register("lib.bbdev");
1115 if (bbdev_logtype >= 0)
1116 rte_log_set_level(bbdev_logtype, RTE_LOG_NOTICE);