replace unused attributes
[dpdk.git] / lib / librte_bbdev / rte_bbdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Intel Corporation
3  */
4
5 #include <stdint.h>
6 #include <string.h>
7 #include <stdbool.h>
8
9 #include <rte_compat.h>
10 #include <rte_common.h>
11 #include <rte_errno.h>
12 #include <rte_log.h>
13 #include <rte_debug.h>
14 #include <rte_eal.h>
15 #include <rte_malloc.h>
16 #include <rte_mempool.h>
17 #include <rte_memzone.h>
18 #include <rte_lcore.h>
19 #include <rte_dev.h>
20 #include <rte_spinlock.h>
21 #include <rte_tailq.h>
22 #include <rte_interrupts.h>
23
24 #include "rte_bbdev_op.h"
25 #include "rte_bbdev.h"
26 #include "rte_bbdev_pmd.h"
27
28 #define DEV_NAME "BBDEV"
29
30
31 /* BBDev library logging ID */
32 static int bbdev_logtype;
33
34 /* Helper macro for logging */
35 #define rte_bbdev_log(level, fmt, ...) \
36         rte_log(RTE_LOG_ ## level, bbdev_logtype, fmt "\n", ##__VA_ARGS__)
37
38 #define rte_bbdev_log_debug(fmt, ...) \
39         rte_bbdev_log(DEBUG, RTE_STR(__LINE__) ":%s() " fmt, __func__, \
40                 ##__VA_ARGS__)
41
42 /* Helper macro to check dev_id is valid */
43 #define VALID_DEV_OR_RET_ERR(dev, dev_id) do { \
44         if (dev == NULL) { \
45                 rte_bbdev_log(ERR, "device %u is invalid", dev_id); \
46                 return -ENODEV; \
47         } \
48 } while (0)
49
50 /* Helper macro to check dev_ops is valid */
51 #define VALID_DEV_OPS_OR_RET_ERR(dev, dev_id) do { \
52         if (dev->dev_ops == NULL) { \
53                 rte_bbdev_log(ERR, "NULL dev_ops structure in device %u", \
54                                 dev_id); \
55                 return -ENODEV; \
56         } \
57 } while (0)
58
59 /* Helper macro to check that driver implements required function pointer */
60 #define VALID_FUNC_OR_RET_ERR(func, dev_id) do { \
61         if (func == NULL) { \
62                 rte_bbdev_log(ERR, "device %u does not support %s", \
63                                 dev_id, #func); \
64                 return -ENOTSUP; \
65         } \
66 } while (0)
67
68 /* Helper macro to check that queue is valid */
69 #define VALID_QUEUE_OR_RET_ERR(queue_id, dev) do { \
70         if (queue_id >= dev->data->num_queues) { \
71                 rte_bbdev_log(ERR, "Invalid queue_id %u for device %u", \
72                                 queue_id, dev->data->dev_id); \
73                 return -ERANGE; \
74         } \
75 } while (0)
76
77 /* List of callback functions registered by an application */
78 struct rte_bbdev_callback {
79         TAILQ_ENTRY(rte_bbdev_callback) next;  /* Callbacks list */
80         rte_bbdev_cb_fn cb_fn;  /* Callback address */
81         void *cb_arg;  /* Parameter for callback */
82         void *ret_param;  /* Return parameter */
83         enum rte_bbdev_event_type event; /* Interrupt event type */
84         uint32_t active; /* Callback is executing */
85 };
86
87 /* spinlock for bbdev device callbacks */
88 static rte_spinlock_t rte_bbdev_cb_lock = RTE_SPINLOCK_INITIALIZER;
89
90 /*
91  * Global array of all devices. This is not static because it's used by the
92  * inline enqueue and dequeue functions
93  */
94 struct rte_bbdev rte_bbdev_devices[RTE_BBDEV_MAX_DEVS];
95
96 /* Global array with rte_bbdev_data structures */
97 static struct rte_bbdev_data *rte_bbdev_data;
98
99 /* Memzone name for global bbdev data pool */
100 static const char *MZ_RTE_BBDEV_DATA = "rte_bbdev_data";
101
102 /* Number of currently valid devices */
103 static uint16_t num_devs;
104
105 /* Return pointer to device structure, with validity check */
106 static struct rte_bbdev *
107 get_dev(uint16_t dev_id)
108 {
109         if (rte_bbdev_is_valid(dev_id))
110                 return &rte_bbdev_devices[dev_id];
111         return NULL;
112 }
113
114 /* Allocate global data array */
115 static int
116 rte_bbdev_data_alloc(void)
117 {
118         const unsigned int flags = 0;
119         const struct rte_memzone *mz;
120
121         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
122                 mz = rte_memzone_reserve(MZ_RTE_BBDEV_DATA,
123                                 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data),
124                                 rte_socket_id(), flags);
125         } else
126                 mz = rte_memzone_lookup(MZ_RTE_BBDEV_DATA);
127         if (mz == NULL) {
128                 rte_bbdev_log(CRIT,
129                                 "Cannot allocate memzone for bbdev port data");
130                 return -ENOMEM;
131         }
132
133         rte_bbdev_data = mz->addr;
134         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
135                 memset(rte_bbdev_data, 0,
136                                 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data));
137         return 0;
138 }
139
140 /*
141  * Find data alocated for the device or if not found return first unused bbdev
142  * data. If all structures are in use and none is used by the device return
143  * NULL.
144  */
145 static struct rte_bbdev_data *
146 find_bbdev_data(const char *name)
147 {
148         uint16_t data_id;
149
150         for (data_id = 0; data_id < RTE_BBDEV_MAX_DEVS; ++data_id) {
151                 if (strlen(rte_bbdev_data[data_id].name) == 0) {
152                         memset(&rte_bbdev_data[data_id], 0,
153                                         sizeof(struct rte_bbdev_data));
154                         return &rte_bbdev_data[data_id];
155                 } else if (strncmp(rte_bbdev_data[data_id].name, name,
156                                 RTE_BBDEV_NAME_MAX_LEN) == 0)
157                         return &rte_bbdev_data[data_id];
158         }
159
160         return NULL;
161 }
162
163 /* Find lowest device id with no attached device */
164 static uint16_t
165 find_free_dev_id(void)
166 {
167         uint16_t i;
168         for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
169                 if (rte_bbdev_devices[i].state == RTE_BBDEV_UNUSED)
170                         return i;
171         }
172         return RTE_BBDEV_MAX_DEVS;
173 }
174
175 struct rte_bbdev *
176 rte_bbdev_allocate(const char *name)
177 {
178         int ret;
179         struct rte_bbdev *bbdev;
180         uint16_t dev_id;
181
182         if (name == NULL) {
183                 rte_bbdev_log(ERR, "Invalid null device name");
184                 return NULL;
185         }
186
187         if (rte_bbdev_get_named_dev(name) != NULL) {
188                 rte_bbdev_log(ERR, "Device \"%s\" is already allocated", name);
189                 return NULL;
190         }
191
192         dev_id = find_free_dev_id();
193         if (dev_id == RTE_BBDEV_MAX_DEVS) {
194                 rte_bbdev_log(ERR, "Reached maximum number of devices");
195                 return NULL;
196         }
197
198         bbdev = &rte_bbdev_devices[dev_id];
199
200         if (rte_bbdev_data == NULL) {
201                 ret = rte_bbdev_data_alloc();
202                 if (ret != 0)
203                         return NULL;
204         }
205
206         bbdev->data = find_bbdev_data(name);
207         if (bbdev->data == NULL) {
208                 rte_bbdev_log(ERR,
209                                 "Max BBDevs already allocated in multi-process environment!");
210                 return NULL;
211         }
212
213         rte_atomic16_inc(&bbdev->data->process_cnt);
214         bbdev->data->dev_id = dev_id;
215         bbdev->state = RTE_BBDEV_INITIALIZED;
216
217         ret = snprintf(bbdev->data->name, RTE_BBDEV_NAME_MAX_LEN, "%s", name);
218         if ((ret < 0) || (ret >= RTE_BBDEV_NAME_MAX_LEN)) {
219                 rte_bbdev_log(ERR, "Copying device name \"%s\" failed", name);
220                 return NULL;
221         }
222
223         /* init user callbacks */
224         TAILQ_INIT(&(bbdev->list_cbs));
225
226         num_devs++;
227
228         rte_bbdev_log_debug("Initialised device %s (id = %u). Num devices = %u",
229                         name, dev_id, num_devs);
230
231         return bbdev;
232 }
233
234 int
235 rte_bbdev_release(struct rte_bbdev *bbdev)
236 {
237         uint16_t dev_id;
238         struct rte_bbdev_callback *cb, *next;
239
240         if (bbdev == NULL) {
241                 rte_bbdev_log(ERR, "NULL bbdev");
242                 return -ENODEV;
243         }
244         dev_id = bbdev->data->dev_id;
245
246         /* free all callbacks from the device's list */
247         for (cb = TAILQ_FIRST(&bbdev->list_cbs); cb != NULL; cb = next) {
248
249                 next = TAILQ_NEXT(cb, next);
250                 TAILQ_REMOVE(&(bbdev->list_cbs), cb, next);
251                 rte_free(cb);
252         }
253
254         /* clear shared BBDev Data if no process is using the device anymore */
255         if (rte_atomic16_dec_and_test(&bbdev->data->process_cnt))
256                 memset(bbdev->data, 0, sizeof(*bbdev->data));
257
258         memset(bbdev, 0, sizeof(*bbdev));
259         num_devs--;
260         bbdev->state = RTE_BBDEV_UNUSED;
261
262         rte_bbdev_log_debug(
263                         "Un-initialised device id = %u. Num devices = %u",
264                         dev_id, num_devs);
265         return 0;
266 }
267
268 struct rte_bbdev *
269 rte_bbdev_get_named_dev(const char *name)
270 {
271         unsigned int i;
272
273         if (name == NULL) {
274                 rte_bbdev_log(ERR, "NULL driver name");
275                 return NULL;
276         }
277
278         for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
279                 struct rte_bbdev *dev = get_dev(i);
280                 if (dev && (strncmp(dev->data->name,
281                                 name, RTE_BBDEV_NAME_MAX_LEN) == 0))
282                         return dev;
283         }
284
285         return NULL;
286 }
287
288 uint16_t
289 rte_bbdev_count(void)
290 {
291         return num_devs;
292 }
293
294 bool
295 rte_bbdev_is_valid(uint16_t dev_id)
296 {
297         if ((dev_id < RTE_BBDEV_MAX_DEVS) &&
298                 rte_bbdev_devices[dev_id].state == RTE_BBDEV_INITIALIZED)
299                 return true;
300         return false;
301 }
302
303 uint16_t
304 rte_bbdev_find_next(uint16_t dev_id)
305 {
306         dev_id++;
307         for (; dev_id < RTE_BBDEV_MAX_DEVS; dev_id++)
308                 if (rte_bbdev_is_valid(dev_id))
309                         break;
310         return dev_id;
311 }
312
313 int
314 rte_bbdev_setup_queues(uint16_t dev_id, uint16_t num_queues, int socket_id)
315 {
316         unsigned int i;
317         int ret;
318         struct rte_bbdev_driver_info dev_info;
319         struct rte_bbdev *dev = get_dev(dev_id);
320         VALID_DEV_OR_RET_ERR(dev, dev_id);
321
322         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
323
324         if (dev->data->started) {
325                 rte_bbdev_log(ERR,
326                                 "Device %u cannot be configured when started",
327                                 dev_id);
328                 return -EBUSY;
329         }
330
331         /* Get device driver information to get max number of queues */
332         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
333         memset(&dev_info, 0, sizeof(dev_info));
334         dev->dev_ops->info_get(dev, &dev_info);
335
336         if ((num_queues == 0) || (num_queues > dev_info.max_num_queues)) {
337                 rte_bbdev_log(ERR,
338                                 "Device %u supports 0 < N <= %u queues, not %u",
339                                 dev_id, dev_info.max_num_queues, num_queues);
340                 return -EINVAL;
341         }
342
343         /* If re-configuration, get driver to free existing internal memory */
344         if (dev->data->queues != NULL) {
345                 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
346                 for (i = 0; i < dev->data->num_queues; i++) {
347                         int ret = dev->dev_ops->queue_release(dev, i);
348                         if (ret < 0) {
349                                 rte_bbdev_log(ERR,
350                                                 "Device %u queue %u release failed",
351                                                 dev_id, i);
352                                 return ret;
353                         }
354                 }
355                 /* Call optional device close */
356                 if (dev->dev_ops->close) {
357                         ret = dev->dev_ops->close(dev);
358                         if (ret < 0) {
359                                 rte_bbdev_log(ERR,
360                                                 "Device %u couldn't be closed",
361                                                 dev_id);
362                                 return ret;
363                         }
364                 }
365                 rte_free(dev->data->queues);
366         }
367
368         /* Allocate queue pointers */
369         dev->data->queues = rte_calloc_socket(DEV_NAME, num_queues,
370                         sizeof(dev->data->queues[0]), RTE_CACHE_LINE_SIZE,
371                                 dev->data->socket_id);
372         if (dev->data->queues == NULL) {
373                 rte_bbdev_log(ERR,
374                                 "calloc of %u queues for device %u on socket %i failed",
375                                 num_queues, dev_id, dev->data->socket_id);
376                 return -ENOMEM;
377         }
378
379         dev->data->num_queues = num_queues;
380
381         /* Call optional device configuration */
382         if (dev->dev_ops->setup_queues) {
383                 ret = dev->dev_ops->setup_queues(dev, num_queues, socket_id);
384                 if (ret < 0) {
385                         rte_bbdev_log(ERR,
386                                         "Device %u memory configuration failed",
387                                         dev_id);
388                         goto error;
389                 }
390         }
391
392         rte_bbdev_log_debug("Device %u set up with %u queues", dev_id,
393                         num_queues);
394         return 0;
395
396 error:
397         dev->data->num_queues = 0;
398         rte_free(dev->data->queues);
399         dev->data->queues = NULL;
400         return ret;
401 }
402
403 int
404 rte_bbdev_intr_enable(uint16_t dev_id)
405 {
406         int ret;
407         struct rte_bbdev *dev = get_dev(dev_id);
408         VALID_DEV_OR_RET_ERR(dev, dev_id);
409
410         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
411
412         if (dev->data->started) {
413                 rte_bbdev_log(ERR,
414                                 "Device %u cannot be configured when started",
415                                 dev_id);
416                 return -EBUSY;
417         }
418
419         if (dev->dev_ops->intr_enable) {
420                 ret = dev->dev_ops->intr_enable(dev);
421                 if (ret < 0) {
422                         rte_bbdev_log(ERR,
423                                         "Device %u interrupts configuration failed",
424                                         dev_id);
425                         return ret;
426                 }
427                 rte_bbdev_log_debug("Enabled interrupts for dev %u", dev_id);
428                 return 0;
429         }
430
431         rte_bbdev_log(ERR, "Device %u doesn't support interrupts", dev_id);
432         return -ENOTSUP;
433 }
434
435 int
436 rte_bbdev_queue_configure(uint16_t dev_id, uint16_t queue_id,
437                 const struct rte_bbdev_queue_conf *conf)
438 {
439         int ret = 0;
440         struct rte_bbdev_driver_info dev_info;
441         struct rte_bbdev *dev = get_dev(dev_id);
442         const struct rte_bbdev_op_cap *p;
443         struct rte_bbdev_queue_conf *stored_conf;
444         const char *op_type_str;
445         VALID_DEV_OR_RET_ERR(dev, dev_id);
446
447         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
448
449         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
450
451         if (dev->data->queues[queue_id].started || dev->data->started) {
452                 rte_bbdev_log(ERR,
453                                 "Queue %u of device %u cannot be configured when started",
454                                 queue_id, dev_id);
455                 return -EBUSY;
456         }
457
458         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
459         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_setup, dev_id);
460
461         /* Get device driver information to verify config is valid */
462         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
463         memset(&dev_info, 0, sizeof(dev_info));
464         dev->dev_ops->info_get(dev, &dev_info);
465
466         /* Check configuration is valid */
467         if (conf != NULL) {
468                 if ((conf->op_type == RTE_BBDEV_OP_NONE) &&
469                                 (dev_info.capabilities[0].type ==
470                                 RTE_BBDEV_OP_NONE)) {
471                         ret = 1;
472                 } else {
473                         for (p = dev_info.capabilities;
474                                         p->type != RTE_BBDEV_OP_NONE; p++) {
475                                 if (conf->op_type == p->type) {
476                                         ret = 1;
477                                         break;
478                                 }
479                         }
480                 }
481                 if (ret == 0) {
482                         rte_bbdev_log(ERR, "Invalid operation type");
483                         return -EINVAL;
484                 }
485                 if (conf->queue_size > dev_info.queue_size_lim) {
486                         rte_bbdev_log(ERR,
487                                         "Size (%u) of queue %u of device %u must be: <= %u",
488                                         conf->queue_size, queue_id, dev_id,
489                                         dev_info.queue_size_lim);
490                         return -EINVAL;
491                 }
492                 if (!rte_is_power_of_2(conf->queue_size)) {
493                         rte_bbdev_log(ERR,
494                                         "Size (%u) of queue %u of device %u must be a power of 2",
495                                         conf->queue_size, queue_id, dev_id);
496                         return -EINVAL;
497                 }
498                 if (conf->op_type == RTE_BBDEV_OP_TURBO_DEC &&
499                         conf->priority > dev_info.max_ul_queue_priority) {
500                         rte_bbdev_log(ERR,
501                                         "Priority (%u) of queue %u of bbdev %u must be <= %u",
502                                         conf->priority, queue_id, dev_id,
503                                         dev_info.max_ul_queue_priority);
504                         return -EINVAL;
505                 }
506                 if (conf->op_type == RTE_BBDEV_OP_TURBO_ENC &&
507                         conf->priority > dev_info.max_dl_queue_priority) {
508                         rte_bbdev_log(ERR,
509                                         "Priority (%u) of queue %u of bbdev %u must be <= %u",
510                                         conf->priority, queue_id, dev_id,
511                                         dev_info.max_dl_queue_priority);
512                         return -EINVAL;
513                 }
514         }
515
516         /* Release existing queue (in case of queue reconfiguration) */
517         if (dev->data->queues[queue_id].queue_private != NULL) {
518                 ret = dev->dev_ops->queue_release(dev, queue_id);
519                 if (ret < 0) {
520                         rte_bbdev_log(ERR, "Device %u queue %u release failed",
521                                         dev_id, queue_id);
522                         return ret;
523                 }
524         }
525
526         /* Get driver to setup the queue */
527         ret = dev->dev_ops->queue_setup(dev, queue_id, (conf != NULL) ?
528                         conf : &dev_info.default_queue_conf);
529         if (ret < 0) {
530                 rte_bbdev_log(ERR,
531                                 "Device %u queue %u setup failed", dev_id,
532                                 queue_id);
533                 return ret;
534         }
535
536         /* Store configuration */
537         stored_conf = &dev->data->queues[queue_id].conf;
538         memcpy(stored_conf,
539                         (conf != NULL) ? conf : &dev_info.default_queue_conf,
540                         sizeof(*stored_conf));
541
542         op_type_str = rte_bbdev_op_type_str(stored_conf->op_type);
543         if (op_type_str == NULL)
544                 return -EINVAL;
545
546         rte_bbdev_log_debug("Configured dev%uq%u (size=%u, type=%s, prio=%u)",
547                         dev_id, queue_id, stored_conf->queue_size, op_type_str,
548                         stored_conf->priority);
549
550         return 0;
551 }
552
553 int
554 rte_bbdev_start(uint16_t dev_id)
555 {
556         int i;
557         struct rte_bbdev *dev = get_dev(dev_id);
558         VALID_DEV_OR_RET_ERR(dev, dev_id);
559
560         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
561
562         if (dev->data->started) {
563                 rte_bbdev_log_debug("Device %u is already started", dev_id);
564                 return 0;
565         }
566
567         if (dev->dev_ops->start) {
568                 int ret = dev->dev_ops->start(dev);
569                 if (ret < 0) {
570                         rte_bbdev_log(ERR, "Device %u start failed", dev_id);
571                         return ret;
572                 }
573         }
574
575         /* Store new state */
576         for (i = 0; i < dev->data->num_queues; i++)
577                 if (!dev->data->queues[i].conf.deferred_start)
578                         dev->data->queues[i].started = true;
579         dev->data->started = true;
580
581         rte_bbdev_log_debug("Started device %u", dev_id);
582         return 0;
583 }
584
585 int
586 rte_bbdev_stop(uint16_t dev_id)
587 {
588         struct rte_bbdev *dev = get_dev(dev_id);
589         VALID_DEV_OR_RET_ERR(dev, dev_id);
590
591         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
592
593         if (!dev->data->started) {
594                 rte_bbdev_log_debug("Device %u is already stopped", dev_id);
595                 return 0;
596         }
597
598         if (dev->dev_ops->stop)
599                 dev->dev_ops->stop(dev);
600         dev->data->started = false;
601
602         rte_bbdev_log_debug("Stopped device %u", dev_id);
603         return 0;
604 }
605
606 int
607 rte_bbdev_close(uint16_t dev_id)
608 {
609         int ret;
610         uint16_t i;
611         struct rte_bbdev *dev = get_dev(dev_id);
612         VALID_DEV_OR_RET_ERR(dev, dev_id);
613
614         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
615
616         if (dev->data->started) {
617                 ret = rte_bbdev_stop(dev_id);
618                 if (ret < 0) {
619                         rte_bbdev_log(ERR, "Device %u stop failed", dev_id);
620                         return ret;
621                 }
622         }
623
624         /* Free memory used by queues */
625         for (i = 0; i < dev->data->num_queues; i++) {
626                 ret = dev->dev_ops->queue_release(dev, i);
627                 if (ret < 0) {
628                         rte_bbdev_log(ERR, "Device %u queue %u release failed",
629                                         dev_id, i);
630                         return ret;
631                 }
632         }
633         rte_free(dev->data->queues);
634
635         if (dev->dev_ops->close) {
636                 ret = dev->dev_ops->close(dev);
637                 if (ret < 0) {
638                         rte_bbdev_log(ERR, "Device %u close failed", dev_id);
639                         return ret;
640                 }
641         }
642
643         /* Clear configuration */
644         dev->data->queues = NULL;
645         dev->data->num_queues = 0;
646
647         rte_bbdev_log_debug("Closed device %u", dev_id);
648         return 0;
649 }
650
651 int
652 rte_bbdev_queue_start(uint16_t dev_id, uint16_t queue_id)
653 {
654         struct rte_bbdev *dev = get_dev(dev_id);
655         VALID_DEV_OR_RET_ERR(dev, dev_id);
656
657         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
658
659         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
660
661         if (dev->data->queues[queue_id].started) {
662                 rte_bbdev_log_debug("Queue %u of device %u already started",
663                                 queue_id, dev_id);
664                 return 0;
665         }
666
667         if (dev->dev_ops->queue_start) {
668                 int ret = dev->dev_ops->queue_start(dev, queue_id);
669                 if (ret < 0) {
670                         rte_bbdev_log(ERR, "Device %u queue %u start failed",
671                                         dev_id, queue_id);
672                         return ret;
673                 }
674         }
675         dev->data->queues[queue_id].started = true;
676
677         rte_bbdev_log_debug("Started queue %u of device %u", queue_id, dev_id);
678         return 0;
679 }
680
681 int
682 rte_bbdev_queue_stop(uint16_t dev_id, uint16_t queue_id)
683 {
684         struct rte_bbdev *dev = get_dev(dev_id);
685         VALID_DEV_OR_RET_ERR(dev, dev_id);
686
687         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
688
689         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
690
691         if (!dev->data->queues[queue_id].started) {
692                 rte_bbdev_log_debug("Queue %u of device %u already stopped",
693                                 queue_id, dev_id);
694                 return 0;
695         }
696
697         if (dev->dev_ops->queue_stop) {
698                 int ret = dev->dev_ops->queue_stop(dev, queue_id);
699                 if (ret < 0) {
700                         rte_bbdev_log(ERR, "Device %u queue %u stop failed",
701                                         dev_id, queue_id);
702                         return ret;
703                 }
704         }
705         dev->data->queues[queue_id].started = false;
706
707         rte_bbdev_log_debug("Stopped queue %u of device %u", queue_id, dev_id);
708         return 0;
709 }
710
711 /* Get device statistics */
712 static void
713 get_stats_from_queues(struct rte_bbdev *dev, struct rte_bbdev_stats *stats)
714 {
715         unsigned int q_id;
716         for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
717                 struct rte_bbdev_stats *q_stats =
718                                 &dev->data->queues[q_id].queue_stats;
719
720                 stats->enqueued_count += q_stats->enqueued_count;
721                 stats->dequeued_count += q_stats->dequeued_count;
722                 stats->enqueue_err_count += q_stats->enqueue_err_count;
723                 stats->dequeue_err_count += q_stats->dequeue_err_count;
724         }
725         rte_bbdev_log_debug("Got stats on %u", dev->data->dev_id);
726 }
727
728 static void
729 reset_stats_in_queues(struct rte_bbdev *dev)
730 {
731         unsigned int q_id;
732         for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
733                 struct rte_bbdev_stats *q_stats =
734                                 &dev->data->queues[q_id].queue_stats;
735
736                 memset(q_stats, 0, sizeof(*q_stats));
737         }
738         rte_bbdev_log_debug("Reset stats on %u", dev->data->dev_id);
739 }
740
741 int
742 rte_bbdev_stats_get(uint16_t dev_id, struct rte_bbdev_stats *stats)
743 {
744         struct rte_bbdev *dev = get_dev(dev_id);
745         VALID_DEV_OR_RET_ERR(dev, dev_id);
746
747         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
748
749         if (stats == NULL) {
750                 rte_bbdev_log(ERR, "NULL stats structure");
751                 return -EINVAL;
752         }
753
754         memset(stats, 0, sizeof(*stats));
755         if (dev->dev_ops->stats_get != NULL)
756                 dev->dev_ops->stats_get(dev, stats);
757         else
758                 get_stats_from_queues(dev, stats);
759
760         rte_bbdev_log_debug("Retrieved stats of device %u", dev_id);
761         return 0;
762 }
763
764 int
765 rte_bbdev_stats_reset(uint16_t dev_id)
766 {
767         struct rte_bbdev *dev = get_dev(dev_id);
768         VALID_DEV_OR_RET_ERR(dev, dev_id);
769
770         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
771
772         if (dev->dev_ops->stats_reset != NULL)
773                 dev->dev_ops->stats_reset(dev);
774         else
775                 reset_stats_in_queues(dev);
776
777         rte_bbdev_log_debug("Reset stats of device %u", dev_id);
778         return 0;
779 }
780
781 int
782 rte_bbdev_info_get(uint16_t dev_id, struct rte_bbdev_info *dev_info)
783 {
784         struct rte_bbdev *dev = get_dev(dev_id);
785         VALID_DEV_OR_RET_ERR(dev, dev_id);
786
787         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
788
789         if (dev_info == NULL) {
790                 rte_bbdev_log(ERR, "NULL dev info structure");
791                 return -EINVAL;
792         }
793
794         /* Copy data maintained by device interface layer */
795         memset(dev_info, 0, sizeof(*dev_info));
796         dev_info->dev_name = dev->data->name;
797         dev_info->num_queues = dev->data->num_queues;
798         dev_info->device = dev->device;
799         dev_info->socket_id = dev->data->socket_id;
800         dev_info->started = dev->data->started;
801
802         /* Copy data maintained by device driver layer */
803         dev->dev_ops->info_get(dev, &dev_info->drv);
804
805         rte_bbdev_log_debug("Retrieved info of device %u", dev_id);
806         return 0;
807 }
808
809 int
810 rte_bbdev_queue_info_get(uint16_t dev_id, uint16_t queue_id,
811                 struct rte_bbdev_queue_info *queue_info)
812 {
813         struct rte_bbdev *dev = get_dev(dev_id);
814         VALID_DEV_OR_RET_ERR(dev, dev_id);
815
816         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
817
818         if (queue_info == NULL) {
819                 rte_bbdev_log(ERR, "NULL queue info structure");
820                 return -EINVAL;
821         }
822
823         /* Copy data to output */
824         memset(queue_info, 0, sizeof(*queue_info));
825         queue_info->conf = dev->data->queues[queue_id].conf;
826         queue_info->started = dev->data->queues[queue_id].started;
827
828         rte_bbdev_log_debug("Retrieved info of queue %u of device %u",
829                         queue_id, dev_id);
830         return 0;
831 }
832
833 /* Calculate size needed to store bbdev_op, depending on type */
834 static unsigned int
835 get_bbdev_op_size(enum rte_bbdev_op_type type)
836 {
837         unsigned int result = 0;
838         switch (type) {
839         case RTE_BBDEV_OP_NONE:
840                 result = RTE_MAX(sizeof(struct rte_bbdev_dec_op),
841                                 sizeof(struct rte_bbdev_enc_op));
842                 break;
843         case RTE_BBDEV_OP_TURBO_DEC:
844                 result = sizeof(struct rte_bbdev_dec_op);
845                 break;
846         case RTE_BBDEV_OP_TURBO_ENC:
847                 result = sizeof(struct rte_bbdev_enc_op);
848                 break;
849         case RTE_BBDEV_OP_LDPC_DEC:
850                 result = sizeof(struct rte_bbdev_dec_op);
851                 break;
852         case RTE_BBDEV_OP_LDPC_ENC:
853                 result = sizeof(struct rte_bbdev_enc_op);
854                 break;
855         default:
856                 break;
857         }
858
859         return result;
860 }
861
862 /* Initialise a bbdev_op structure */
863 static void
864 bbdev_op_init(struct rte_mempool *mempool, void *arg, void *element,
865                 __rte_unused unsigned int n)
866 {
867         enum rte_bbdev_op_type type = *(enum rte_bbdev_op_type *)arg;
868
869         if (type == RTE_BBDEV_OP_TURBO_DEC || type == RTE_BBDEV_OP_LDPC_DEC) {
870                 struct rte_bbdev_dec_op *op = element;
871                 memset(op, 0, mempool->elt_size);
872                 op->mempool = mempool;
873         } else if (type == RTE_BBDEV_OP_TURBO_ENC ||
874                         type == RTE_BBDEV_OP_LDPC_ENC) {
875                 struct rte_bbdev_enc_op *op = element;
876                 memset(op, 0, mempool->elt_size);
877                 op->mempool = mempool;
878         }
879 }
880
881 struct rte_mempool *
882 rte_bbdev_op_pool_create(const char *name, enum rte_bbdev_op_type type,
883                 unsigned int num_elements, unsigned int cache_size,
884                 int socket_id)
885 {
886         struct rte_bbdev_op_pool_private *priv;
887         struct rte_mempool *mp;
888         const char *op_type_str;
889
890         if (name == NULL) {
891                 rte_bbdev_log(ERR, "NULL name for op pool");
892                 return NULL;
893         }
894
895         if (type >= RTE_BBDEV_OP_TYPE_COUNT) {
896                 rte_bbdev_log(ERR,
897                                 "Invalid op type (%u), should be less than %u",
898                                 type, RTE_BBDEV_OP_TYPE_COUNT);
899                 return NULL;
900         }
901
902         mp = rte_mempool_create(name, num_elements, get_bbdev_op_size(type),
903                         cache_size, sizeof(struct rte_bbdev_op_pool_private),
904                         NULL, NULL, bbdev_op_init, &type, socket_id, 0);
905         if (mp == NULL) {
906                 rte_bbdev_log(ERR,
907                                 "Failed to create op pool %s (num ops=%u, op size=%u) with error: %s",
908                                 name, num_elements, get_bbdev_op_size(type),
909                                 rte_strerror(rte_errno));
910                 return NULL;
911         }
912
913         op_type_str = rte_bbdev_op_type_str(type);
914         if (op_type_str == NULL)
915                 return NULL;
916
917         rte_bbdev_log_debug(
918                         "Op pool %s created for %u ops (type=%s, cache=%u, socket=%u, size=%u)",
919                         name, num_elements, op_type_str, cache_size, socket_id,
920                         get_bbdev_op_size(type));
921
922         priv = (struct rte_bbdev_op_pool_private *)rte_mempool_get_priv(mp);
923         priv->type = type;
924
925         return mp;
926 }
927
928 int
929 rte_bbdev_callback_register(uint16_t dev_id, enum rte_bbdev_event_type event,
930                 rte_bbdev_cb_fn cb_fn, void *cb_arg)
931 {
932         struct rte_bbdev_callback *user_cb;
933         struct rte_bbdev *dev = get_dev(dev_id);
934         VALID_DEV_OR_RET_ERR(dev, dev_id);
935
936         if (event >= RTE_BBDEV_EVENT_MAX) {
937                 rte_bbdev_log(ERR,
938                                 "Invalid event type (%u), should be less than %u",
939                                 event, RTE_BBDEV_EVENT_MAX);
940                 return -EINVAL;
941         }
942
943         if (cb_fn == NULL) {
944                 rte_bbdev_log(ERR, "NULL callback function");
945                 return -EINVAL;
946         }
947
948         rte_spinlock_lock(&rte_bbdev_cb_lock);
949
950         TAILQ_FOREACH(user_cb, &(dev->list_cbs), next) {
951                 if (user_cb->cb_fn == cb_fn &&
952                                 user_cb->cb_arg == cb_arg &&
953                                 user_cb->event == event)
954                         break;
955         }
956
957         /* create a new callback. */
958         if (user_cb == NULL) {
959                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
960                                 sizeof(struct rte_bbdev_callback), 0);
961                 if (user_cb != NULL) {
962                         user_cb->cb_fn = cb_fn;
963                         user_cb->cb_arg = cb_arg;
964                         user_cb->event = event;
965                         TAILQ_INSERT_TAIL(&(dev->list_cbs), user_cb, next);
966                 }
967         }
968
969         rte_spinlock_unlock(&rte_bbdev_cb_lock);
970         return (user_cb == NULL) ? -ENOMEM : 0;
971 }
972
973 int
974 rte_bbdev_callback_unregister(uint16_t dev_id, enum rte_bbdev_event_type event,
975                 rte_bbdev_cb_fn cb_fn, void *cb_arg)
976 {
977         int ret = 0;
978         struct rte_bbdev_callback *cb, *next;
979         struct rte_bbdev *dev = get_dev(dev_id);
980         VALID_DEV_OR_RET_ERR(dev, dev_id);
981
982         if (event >= RTE_BBDEV_EVENT_MAX) {
983                 rte_bbdev_log(ERR,
984                                 "Invalid event type (%u), should be less than %u",
985                                 event, RTE_BBDEV_EVENT_MAX);
986                 return -EINVAL;
987         }
988
989         if (cb_fn == NULL) {
990                 rte_bbdev_log(ERR,
991                                 "NULL callback function cannot be unregistered");
992                 return -EINVAL;
993         }
994
995         dev = &rte_bbdev_devices[dev_id];
996         rte_spinlock_lock(&rte_bbdev_cb_lock);
997
998         for (cb = TAILQ_FIRST(&dev->list_cbs); cb != NULL; cb = next) {
999
1000                 next = TAILQ_NEXT(cb, next);
1001
1002                 if (cb->cb_fn != cb_fn || cb->event != event ||
1003                                 (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
1004                         continue;
1005
1006                 /* If this callback is not executing right now, remove it. */
1007                 if (cb->active == 0) {
1008                         TAILQ_REMOVE(&(dev->list_cbs), cb, next);
1009                         rte_free(cb);
1010                 } else
1011                         ret = -EAGAIN;
1012         }
1013
1014         rte_spinlock_unlock(&rte_bbdev_cb_lock);
1015         return ret;
1016 }
1017
1018 void
1019 rte_bbdev_pmd_callback_process(struct rte_bbdev *dev,
1020         enum rte_bbdev_event_type event, void *ret_param)
1021 {
1022         struct rte_bbdev_callback *cb_lst;
1023         struct rte_bbdev_callback dev_cb;
1024
1025         if (dev == NULL) {
1026                 rte_bbdev_log(ERR, "NULL device");
1027                 return;
1028         }
1029
1030         if (dev->data == NULL) {
1031                 rte_bbdev_log(ERR, "NULL data structure");
1032                 return;
1033         }
1034
1035         if (event >= RTE_BBDEV_EVENT_MAX) {
1036                 rte_bbdev_log(ERR,
1037                                 "Invalid event type (%u), should be less than %u",
1038                                 event, RTE_BBDEV_EVENT_MAX);
1039                 return;
1040         }
1041
1042         rte_spinlock_lock(&rte_bbdev_cb_lock);
1043         TAILQ_FOREACH(cb_lst, &(dev->list_cbs), next) {
1044                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1045                         continue;
1046                 dev_cb = *cb_lst;
1047                 cb_lst->active = 1;
1048                 if (ret_param != NULL)
1049                         dev_cb.ret_param = ret_param;
1050
1051                 rte_spinlock_unlock(&rte_bbdev_cb_lock);
1052                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1053                                 dev_cb.cb_arg, dev_cb.ret_param);
1054                 rte_spinlock_lock(&rte_bbdev_cb_lock);
1055                 cb_lst->active = 0;
1056         }
1057         rte_spinlock_unlock(&rte_bbdev_cb_lock);
1058 }
1059
1060 int
1061 rte_bbdev_queue_intr_enable(uint16_t dev_id, uint16_t queue_id)
1062 {
1063         struct rte_bbdev *dev = get_dev(dev_id);
1064         VALID_DEV_OR_RET_ERR(dev, dev_id);
1065         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1066         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1067         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_enable, dev_id);
1068         return dev->dev_ops->queue_intr_enable(dev, queue_id);
1069 }
1070
1071 int
1072 rte_bbdev_queue_intr_disable(uint16_t dev_id, uint16_t queue_id)
1073 {
1074         struct rte_bbdev *dev = get_dev(dev_id);
1075         VALID_DEV_OR_RET_ERR(dev, dev_id);
1076         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1077         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1078         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_disable, dev_id);
1079         return dev->dev_ops->queue_intr_disable(dev, queue_id);
1080 }
1081
1082 int
1083 rte_bbdev_queue_intr_ctl(uint16_t dev_id, uint16_t queue_id, int epfd, int op,
1084                 void *data)
1085 {
1086         uint32_t vec;
1087         struct rte_bbdev *dev = get_dev(dev_id);
1088         struct rte_intr_handle *intr_handle;
1089         int ret;
1090
1091         VALID_DEV_OR_RET_ERR(dev, dev_id);
1092         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1093
1094         intr_handle = dev->intr_handle;
1095         if (!intr_handle || !intr_handle->intr_vec) {
1096                 rte_bbdev_log(ERR, "Device %u intr handle unset\n", dev_id);
1097                 return -ENOTSUP;
1098         }
1099
1100         if (queue_id >= RTE_MAX_RXTX_INTR_VEC_ID) {
1101                 rte_bbdev_log(ERR, "Device %u queue_id %u is too big\n",
1102                                 dev_id, queue_id);
1103                 return -ENOTSUP;
1104         }
1105
1106         vec = intr_handle->intr_vec[queue_id];
1107         ret = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
1108         if (ret && (ret != -EEXIST)) {
1109                 rte_bbdev_log(ERR,
1110                                 "dev %u q %u int ctl error op %d epfd %d vec %u\n",
1111                                 dev_id, queue_id, op, epfd, vec);
1112                 return ret;
1113         }
1114
1115         return 0;
1116 }
1117
1118
1119 const char *
1120 rte_bbdev_op_type_str(enum rte_bbdev_op_type op_type)
1121 {
1122         static const char * const op_types[] = {
1123                 "RTE_BBDEV_OP_NONE",
1124                 "RTE_BBDEV_OP_TURBO_DEC",
1125                 "RTE_BBDEV_OP_TURBO_ENC",
1126                 "RTE_BBDEV_OP_LDPC_DEC",
1127                 "RTE_BBDEV_OP_LDPC_ENC",
1128         };
1129
1130         if (op_type < RTE_BBDEV_OP_TYPE_COUNT)
1131                 return op_types[op_type];
1132
1133         rte_bbdev_log(ERR, "Invalid operation type");
1134         return NULL;
1135 }
1136
1137 RTE_INIT(rte_bbdev_init_log)
1138 {
1139         bbdev_logtype = rte_log_register("lib.bbdev");
1140         if (bbdev_logtype >= 0)
1141                 rte_log_set_level(bbdev_logtype, RTE_LOG_NOTICE);
1142 }