lib/librte_bbdev/rte_bbdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12 #include <rte_debug.h>
  13 #include <rte_eal.h>
  14 #include <rte_malloc.h>
  15 #include <rte_mempool.h>
  16 #include <rte_memzone.h>
  17 #include <rte_lcore.h>
  18 #include <rte_dev.h>
  19 #include <rte_spinlock.h>
  20 #include <rte_tailq.h>
  21 #include <rte_interrupts.h>
  22
  23 #include "rte_bbdev_op.h"
  24 #include "rte_bbdev.h"
  25 #include "rte_bbdev_pmd.h"
  26
  27 #define DEV_NAME "BBDEV"
  28
  29
  30 /* Helper macro to check dev_id is valid */
  31 #define VALID_DEV_OR_RET_ERR(dev, dev_id) do { \
  32         if (dev == NULL) { \
  33                 rte_bbdev_log(ERR, "device %u is invalid", dev_id); \
  34                 return -ENODEV; \
  35         } \
  36 } while (0)
  37
  38 /* Helper macro to check dev_ops is valid */
  39 #define VALID_DEV_OPS_OR_RET_ERR(dev, dev_id) do { \
  40         if (dev->dev_ops == NULL) { \
  41                 rte_bbdev_log(ERR, "NULL dev_ops structure in device %u", \
  42                                 dev_id); \
  43                 return -ENODEV; \
  44         } \
  45 } while (0)
  46
  47 /* Helper macro to check that driver implements required function pointer */
  48 #define VALID_FUNC_OR_RET_ERR(func, dev_id) do { \
  49         if (func == NULL) { \
  50                 rte_bbdev_log(ERR, "device %u does not support %s", \
  51                                 dev_id, #func); \
  52                 return -ENOTSUP; \
  53         } \
  54 } while (0)
  55
  56 /* Helper macro to check that queue is valid */
  57 #define VALID_QUEUE_OR_RET_ERR(queue_id, dev) do { \
  58         if (queue_id >= dev->data->num_queues) { \
  59                 rte_bbdev_log(ERR, "Invalid queue_id %u for device %u", \
  60                                 queue_id, dev->data->dev_id); \
  61                 return -ERANGE; \
  62         } \
  63 } while (0)
  64
  65 /* List of callback functions registered by an application */
  66 struct rte_bbdev_callback {
  67         TAILQ_ENTRY(rte_bbdev_callback) next;  /* Callbacks list */
  68         rte_bbdev_cb_fn cb_fn;  /* Callback address */
  69         void *cb_arg;  /* Parameter for callback */
  70         void *ret_param;  /* Return parameter */
  71         enum rte_bbdev_event_type event; /* Interrupt event type */
  72         uint32_t active; /* Callback is executing */
  73 };
  74
  75 /* spinlock for bbdev device callbacks */
  76 static rte_spinlock_t rte_bbdev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  77
  78 /*
  79  * Global array of all devices. This is not static because it's used by the
  80  * inline enqueue and dequeue functions
  81  */
  82 struct rte_bbdev rte_bbdev_devices[RTE_BBDEV_MAX_DEVS];
  83
  84 /* Global array with rte_bbdev_data structures */
  85 static struct rte_bbdev_data *rte_bbdev_data;
  86
  87 /* Memzone name for global bbdev data pool */
  88 static const char *MZ_RTE_BBDEV_DATA = "rte_bbdev_data";
  89
  90 /* Number of currently valid devices */
  91 static uint16_t num_devs;
  92
  93 /* Return pointer to device structure, with validity check */
  94 static struct rte_bbdev *
  95 get_dev(uint16_t dev_id)
  96 {
  97         if (rte_bbdev_is_valid(dev_id))
  98                 return &rte_bbdev_devices[dev_id];
  99         return NULL;
 100 }
 101
 102 /* Allocate global data array */
 103 static int
 104 rte_bbdev_data_alloc(void)
 105 {
 106         const unsigned int flags = 0;
 107         const struct rte_memzone *mz;
 108
 109         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 110                 mz = rte_memzone_reserve(MZ_RTE_BBDEV_DATA,
 111                                 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data),
 112                                 rte_socket_id(), flags);
 113         } else
 114                 mz = rte_memzone_lookup(MZ_RTE_BBDEV_DATA);
 115         if (mz == NULL) {
 116                 rte_bbdev_log(CRIT,
 117                                 "Cannot allocate memzone for bbdev port data");
 118                 return -ENOMEM;
 119         }
 120
 121         rte_bbdev_data = mz->addr;
 122         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 123                 memset(rte_bbdev_data, 0,
 124                                 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data));
 125         return 0;
 126 }
 127
 128 /*
 129  * Find data alocated for the device or if not found return first unused bbdev
 130  * data. If all structures are in use and none is used by the device return
 131  * NULL.
 132  */
 133 static struct rte_bbdev_data *
 134 find_bbdev_data(const char *name)
 135 {
 136         uint16_t data_id;
 137
 138         for (data_id = 0; data_id < RTE_BBDEV_MAX_DEVS; ++data_id) {
 139                 if (strlen(rte_bbdev_data[data_id].name) == 0) {
 140                         memset(&rte_bbdev_data[data_id], 0,
 141                                         sizeof(struct rte_bbdev_data));
 142                         return &rte_bbdev_data[data_id];
 143                 } else if (strncmp(rte_bbdev_data[data_id].name, name,
 144                                 RTE_BBDEV_NAME_MAX_LEN) == 0)
 145                         return &rte_bbdev_data[data_id];
 146         }
 147
 148         return NULL;
 149 }
 150
 151 /* Find lowest device id with no attached device */
 152 static uint16_t
 153 find_free_dev_id(void)
 154 {
 155         uint16_t i;
 156         for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
 157                 if (rte_bbdev_devices[i].state == RTE_BBDEV_UNUSED)
 158                         return i;
 159         }
 160         return RTE_BBDEV_MAX_DEVS;
 161 }
 162
 163 struct rte_bbdev *
 164 rte_bbdev_allocate(const char *name)
 165 {
 166         int ret;
 167         struct rte_bbdev *bbdev;
 168         uint16_t dev_id;
 169
 170         if (name == NULL) {
 171                 rte_bbdev_log(ERR, "Invalid null device name");
 172                 return NULL;
 173         }
 174
 175         if (rte_bbdev_get_named_dev(name) != NULL) {
 176                 rte_bbdev_log(ERR, "Device \"%s\" is already allocated", name);
 177                 return NULL;
 178         }
 179
 180         dev_id = find_free_dev_id();
 181         if (dev_id == RTE_BBDEV_MAX_DEVS) {
 182                 rte_bbdev_log(ERR, "Reached maximum number of devices");
 183                 return NULL;
 184         }
 185
 186         bbdev = &rte_bbdev_devices[dev_id];
 187
 188         if (rte_bbdev_data == NULL) {
 189                 ret = rte_bbdev_data_alloc();
 190                 if (ret != 0)
 191                         return NULL;
 192         }
 193
 194         bbdev->data = find_bbdev_data(name);
 195         if (bbdev->data == NULL) {
 196                 rte_bbdev_log(ERR,
 197                                 "Max BBDevs already allocated in multi-process environment!");
 198                 return NULL;
 199         }
 200
 201         rte_atomic16_inc(&bbdev->data->process_cnt);
 202         bbdev->data->dev_id = dev_id;
 203         bbdev->state = RTE_BBDEV_INITIALIZED;
 204
 205         ret = snprintf(bbdev->data->name, RTE_BBDEV_NAME_MAX_LEN, "%s", name);
 206         if ((ret < 0) || (ret >= RTE_BBDEV_NAME_MAX_LEN)) {
 207                 rte_bbdev_log(ERR, "Copying device name \"%s\" failed", name);
 208                 return NULL;
 209         }
 210
 211         /* init user callbacks */
 212         TAILQ_INIT(&(bbdev->list_cbs));
 213
 214         num_devs++;
 215
 216         rte_bbdev_log_debug("Initialised device %s (id = %u). Num devices = %u",
 217                         name, dev_id, num_devs);
 218
 219         return bbdev;
 220 }
 221
 222 int
 223 rte_bbdev_release(struct rte_bbdev *bbdev)
 224 {
 225         uint16_t dev_id;
 226         struct rte_bbdev_callback *cb, *next;
 227
 228         if (bbdev == NULL) {
 229                 rte_bbdev_log(ERR, "NULL bbdev");
 230                 return -ENODEV;
 231         }
 232         dev_id = bbdev->data->dev_id;
 233
 234         /* free all callbacks from the device's list */
 235         for (cb = TAILQ_FIRST(&bbdev->list_cbs); cb != NULL; cb = next) {
 236
 237                 next = TAILQ_NEXT(cb, next);
 238                 TAILQ_REMOVE(&(bbdev->list_cbs), cb, next);
 239                 rte_free(cb);
 240         }
 241
 242         /* clear shared BBDev Data if no process is using the device anymore */
 243         if (rte_atomic16_dec_and_test(&bbdev->data->process_cnt))
 244                 memset(bbdev->data, 0, sizeof(*bbdev->data));
 245
 246         memset(bbdev, 0, sizeof(*bbdev));
 247         num_devs--;
 248         bbdev->state = RTE_BBDEV_UNUSED;
 249
 250         rte_bbdev_log_debug(
 251                         "Un-initialised device id = %u. Num devices = %u",
 252                         dev_id, num_devs);
 253         return 0;
 254 }
 255
 256 struct rte_bbdev *
 257 rte_bbdev_get_named_dev(const char *name)
 258 {
 259         unsigned int i;
 260
 261         if (name == NULL) {
 262                 rte_bbdev_log(ERR, "NULL driver name");
 263                 return NULL;
 264         }
 265
 266         for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
 267                 struct rte_bbdev *dev = get_dev(i);
 268                 if (dev && (strncmp(dev->data->name,
 269                                 name, RTE_BBDEV_NAME_MAX_LEN) == 0))
 270                         return dev;
 271         }
 272
 273         return NULL;
 274 }
 275
 276 uint16_t
 277 rte_bbdev_count(void)
 278 {
 279         return num_devs;
 280 }
 281
 282 bool
 283 rte_bbdev_is_valid(uint16_t dev_id)
 284 {
 285         if ((dev_id < RTE_BBDEV_MAX_DEVS) &&
 286                 rte_bbdev_devices[dev_id].state == RTE_BBDEV_INITIALIZED)
 287                 return true;
 288         return false;
 289 }
 290
 291 uint16_t
 292 rte_bbdev_find_next(uint16_t dev_id)
 293 {
 294         dev_id++;
 295         for (; dev_id < RTE_BBDEV_MAX_DEVS; dev_id++)
 296                 if (rte_bbdev_is_valid(dev_id))
 297                         break;
 298         return dev_id;
 299 }
 300
 301 int
 302 rte_bbdev_setup_queues(uint16_t dev_id, uint16_t num_queues, int socket_id)
 303 {
 304         unsigned int i;
 305         int ret;
 306         struct rte_bbdev_driver_info dev_info;
 307         struct rte_bbdev *dev = get_dev(dev_id);
 308         VALID_DEV_OR_RET_ERR(dev, dev_id);
 309
 310         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 311
 312         if (dev->data->started) {
 313                 rte_bbdev_log(ERR,
 314                                 "Device %u cannot be configured when started",
 315                                 dev_id);
 316                 return -EBUSY;
 317         }
 318
 319         /* Get device driver information to get max number of queues */
 320         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 321         memset(&dev_info, 0, sizeof(dev_info));
 322         dev->dev_ops->info_get(dev, &dev_info);
 323
 324         if ((num_queues == 0) || (num_queues > dev_info.max_num_queues)) {
 325                 rte_bbdev_log(ERR,
 326                                 "Device %u supports 0 < N <= %u queues, not %u",
 327                                 dev_id, dev_info.max_num_queues, num_queues);
 328                 return -EINVAL;
 329         }
 330
 331         /* If re-configuration, get driver to free existing internal memory */
 332         if (dev->data->queues != NULL) {
 333                 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
 334                 for (i = 0; i < dev->data->num_queues; i++) {
 335                         int ret = dev->dev_ops->queue_release(dev, i);
 336                         if (ret < 0) {
 337                                 rte_bbdev_log(ERR,
 338                                                 "Device %u queue %u release failed",
 339                                                 dev_id, i);
 340                                 return ret;
 341                         }
 342                 }
 343                 /* Call optional device close */
 344                 if (dev->dev_ops->close) {
 345                         ret = dev->dev_ops->close(dev);
 346                         if (ret < 0) {
 347                                 rte_bbdev_log(ERR,
 348                                                 "Device %u couldn't be closed",
 349                                                 dev_id);
 350                                 return ret;
 351                         }
 352                 }
 353                 rte_free(dev->data->queues);
 354         }
 355
 356         /* Allocate queue pointers */
 357         dev->data->queues = rte_calloc_socket(DEV_NAME, num_queues,
 358                         sizeof(dev->data->queues[0]), RTE_CACHE_LINE_SIZE,
 359                                 dev->data->socket_id);
 360         if (dev->data->queues == NULL) {
 361                 rte_bbdev_log(ERR,
 362                                 "calloc of %u queues for device %u on socket %i failed",
 363                                 num_queues, dev_id, dev->data->socket_id);
 364                 return -ENOMEM;
 365         }
 366
 367         dev->data->num_queues = num_queues;
 368
 369         /* Call optional device configuration */
 370         if (dev->dev_ops->setup_queues) {
 371                 ret = dev->dev_ops->setup_queues(dev, num_queues, socket_id);
 372                 if (ret < 0) {
 373                         rte_bbdev_log(ERR,
 374                                         "Device %u memory configuration failed",
 375                                         dev_id);
 376                         goto error;
 377                 }
 378         }
 379
 380         rte_bbdev_log_debug("Device %u set up with %u queues", dev_id,
 381                         num_queues);
 382         return 0;
 383
 384 error:
 385         dev->data->num_queues = 0;
 386         rte_free(dev->data->queues);
 387         dev->data->queues = NULL;
 388         return ret;
 389 }
 390
 391 int
 392 rte_bbdev_intr_enable(uint16_t dev_id)
 393 {
 394         int ret;
 395         struct rte_bbdev *dev = get_dev(dev_id);
 396         VALID_DEV_OR_RET_ERR(dev, dev_id);
 397
 398         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 399
 400         if (dev->data->started) {
 401                 rte_bbdev_log(ERR,
 402                                 "Device %u cannot be configured when started",
 403                                 dev_id);
 404                 return -EBUSY;
 405         }
 406
 407         if (dev->dev_ops->intr_enable) {
 408                 ret = dev->dev_ops->intr_enable(dev);
 409                 if (ret < 0) {
 410                         rte_bbdev_log(ERR,
 411                                         "Device %u interrupts configuration failed",
 412                                         dev_id);
 413                         return ret;
 414                 }
 415                 rte_bbdev_log_debug("Enabled interrupts for dev %u", dev_id);
 416                 return 0;
 417         }
 418
 419         rte_bbdev_log(ERR, "Device %u doesn't support interrupts", dev_id);
 420         return -ENOTSUP;
 421 }
 422
 423 int
 424 rte_bbdev_queue_configure(uint16_t dev_id, uint16_t queue_id,
 425                 const struct rte_bbdev_queue_conf *conf)
 426 {
 427         int ret = 0;
 428         struct rte_bbdev_driver_info dev_info;
 429         struct rte_bbdev *dev = get_dev(dev_id);
 430         const struct rte_bbdev_op_cap *p;
 431         struct rte_bbdev_queue_conf *stored_conf;
 432         const char *op_type_str;
 433         VALID_DEV_OR_RET_ERR(dev, dev_id);
 434
 435         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 436
 437         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 438
 439         if (dev->data->queues[queue_id].started || dev->data->started) {
 440                 rte_bbdev_log(ERR,
 441                                 "Queue %u of device %u cannot be configured when started",
 442                                 queue_id, dev_id);
 443                 return -EBUSY;
 444         }
 445
 446         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
 447         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_setup, dev_id);
 448
 449         /* Get device driver information to verify config is valid */
 450         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 451         memset(&dev_info, 0, sizeof(dev_info));
 452         dev->dev_ops->info_get(dev, &dev_info);
 453
 454         /* Check configuration is valid */
 455         if (conf != NULL) {
 456                 if ((conf->op_type == RTE_BBDEV_OP_NONE) &&
 457                                 (dev_info.capabilities[0].type ==
 458                                 RTE_BBDEV_OP_NONE)) {
 459                         ret = 1;
 460                 } else {
 461                         for (p = dev_info.capabilities;
 462                                         p->type != RTE_BBDEV_OP_NONE; p++) {
 463                                 if (conf->op_type == p->type) {
 464                                         ret = 1;
 465                                         break;
 466                                 }
 467                         }
 468                 }
 469                 if (ret == 0) {
 470                         rte_bbdev_log(ERR, "Invalid operation type");
 471                         return -EINVAL;
 472                 }
 473                 if (conf->queue_size > dev_info.queue_size_lim) {
 474                         rte_bbdev_log(ERR,
 475                                         "Size (%u) of queue %u of device %u must be: <= %u",
 476                                         conf->queue_size, queue_id, dev_id,
 477                                         dev_info.queue_size_lim);
 478                         return -EINVAL;
 479                 }
 480                 if (!rte_is_power_of_2(conf->queue_size)) {
 481                         rte_bbdev_log(ERR,
 482                                         "Size (%u) of queue %u of device %u must be a power of 2",
 483                                         conf->queue_size, queue_id, dev_id);
 484                         return -EINVAL;
 485                 }
 486                 if (conf->priority > dev_info.max_queue_priority) {
 487                         rte_bbdev_log(ERR,
 488                                         "Priority (%u) of queue %u of bdev %u must be <= %u",
 489                                         conf->priority, queue_id, dev_id,
 490                                         dev_info.max_queue_priority);
 491                         return -EINVAL;
 492                 }
 493         }
 494
 495         /* Release existing queue (in case of queue reconfiguration) */
 496         if (dev->data->queues[queue_id].queue_private != NULL) {
 497                 ret = dev->dev_ops->queue_release(dev, queue_id);
 498                 if (ret < 0) {
 499                         rte_bbdev_log(ERR, "Device %u queue %u release failed",
 500                                         dev_id, queue_id);
 501                         return ret;
 502                 }
 503         }
 504
 505         /* Get driver to setup the queue */
 506         ret = dev->dev_ops->queue_setup(dev, queue_id, (conf != NULL) ?
 507                         conf : &dev_info.default_queue_conf);
 508         if (ret < 0) {
 509                 rte_bbdev_log(ERR,
 510                                 "Device %u queue %u setup failed", dev_id,
 511                                 queue_id);
 512                 return ret;
 513         }
 514
 515         /* Store configuration */
 516         stored_conf = &dev->data->queues[queue_id].conf;
 517         memcpy(stored_conf,
 518                         (conf != NULL) ? conf : &dev_info.default_queue_conf,
 519                         sizeof(*stored_conf));
 520
 521         op_type_str = rte_bbdev_op_type_str(stored_conf->op_type);
 522         if (op_type_str == NULL)
 523                 return -EINVAL;
 524
 525         rte_bbdev_log_debug("Configured dev%uq%u (size=%u, type=%s, prio=%u)",
 526                         dev_id, queue_id, stored_conf->queue_size, op_type_str,
 527                         stored_conf->priority);
 528
 529         return 0;
 530 }
 531
 532 int
 533 rte_bbdev_start(uint16_t dev_id)
 534 {
 535         int i;
 536         struct rte_bbdev *dev = get_dev(dev_id);
 537         VALID_DEV_OR_RET_ERR(dev, dev_id);
 538
 539         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 540
 541         if (dev->data->started) {
 542                 rte_bbdev_log_debug("Device %u is already started", dev_id);
 543                 return 0;
 544         }
 545
 546         if (dev->dev_ops->start) {
 547                 int ret = dev->dev_ops->start(dev);
 548                 if (ret < 0) {
 549                         rte_bbdev_log(ERR, "Device %u start failed", dev_id);
 550                         return ret;
 551                 }
 552         }
 553
 554         /* Store new state */
 555         for (i = 0; i < dev->data->num_queues; i++)
 556                 if (!dev->data->queues[i].conf.deferred_start)
 557                         dev->data->queues[i].started = true;
 558         dev->data->started = true;
 559
 560         rte_bbdev_log_debug("Started device %u", dev_id);
 561         return 0;
 562 }
 563
 564 int
 565 rte_bbdev_stop(uint16_t dev_id)
 566 {
 567         struct rte_bbdev *dev = get_dev(dev_id);
 568         VALID_DEV_OR_RET_ERR(dev, dev_id);
 569
 570         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 571
 572         if (!dev->data->started) {
 573                 rte_bbdev_log_debug("Device %u is already stopped", dev_id);
 574                 return 0;
 575         }
 576
 577         if (dev->dev_ops->stop)
 578                 dev->dev_ops->stop(dev);
 579         dev->data->started = false;
 580
 581         rte_bbdev_log_debug("Stopped device %u", dev_id);
 582         return 0;
 583 }
 584
 585 int
 586 rte_bbdev_close(uint16_t dev_id)
 587 {
 588         int ret;
 589         uint16_t i;
 590         struct rte_bbdev *dev = get_dev(dev_id);
 591         VALID_DEV_OR_RET_ERR(dev, dev_id);
 592
 593         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 594
 595         if (dev->data->started) {
 596                 ret = rte_bbdev_stop(dev_id);
 597                 if (ret < 0) {
 598                         rte_bbdev_log(ERR, "Device %u stop failed", dev_id);
 599                         return ret;
 600                 }
 601         }
 602
 603         /* Free memory used by queues */
 604         for (i = 0; i < dev->data->num_queues; i++) {
 605                 ret = dev->dev_ops->queue_release(dev, i);
 606                 if (ret < 0) {
 607                         rte_bbdev_log(ERR, "Device %u queue %u release failed",
 608                                         dev_id, i);
 609                         return ret;
 610                 }
 611         }
 612         rte_free(dev->data->queues);
 613
 614         if (dev->dev_ops->close) {
 615                 ret = dev->dev_ops->close(dev);
 616                 if (ret < 0) {
 617                         rte_bbdev_log(ERR, "Device %u close failed", dev_id);
 618                         return ret;
 619                 }
 620         }
 621
 622         /* Clear configuration */
 623         dev->data->queues = NULL;
 624         dev->data->num_queues = 0;
 625
 626         rte_bbdev_log_debug("Closed device %u", dev_id);
 627         return 0;
 628 }
 629
 630 int
 631 rte_bbdev_queue_start(uint16_t dev_id, uint16_t queue_id)
 632 {
 633         struct rte_bbdev *dev = get_dev(dev_id);
 634         VALID_DEV_OR_RET_ERR(dev, dev_id);
 635
 636         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 637
 638         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 639
 640         if (dev->data->queues[queue_id].started) {
 641                 rte_bbdev_log_debug("Queue %u of device %u already started",
 642                                 queue_id, dev_id);
 643                 return 0;
 644         }
 645
 646         if (dev->dev_ops->queue_start) {
 647                 int ret = dev->dev_ops->queue_start(dev, queue_id);
 648                 if (ret < 0) {
 649                         rte_bbdev_log(ERR, "Device %u queue %u start failed",
 650                                         dev_id, queue_id);
 651                         return ret;
 652                 }
 653         }
 654         dev->data->queues[queue_id].started = true;
 655
 656         rte_bbdev_log_debug("Started queue %u of device %u", queue_id, dev_id);
 657         return 0;
 658 }
 659
 660 int
 661 rte_bbdev_queue_stop(uint16_t dev_id, uint16_t queue_id)
 662 {
 663         struct rte_bbdev *dev = get_dev(dev_id);
 664         VALID_DEV_OR_RET_ERR(dev, dev_id);
 665
 666         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 667
 668         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 669
 670         if (!dev->data->queues[queue_id].started) {
 671                 rte_bbdev_log_debug("Queue %u of device %u already stopped",
 672                                 queue_id, dev_id);
 673                 return 0;
 674         }
 675
 676         if (dev->dev_ops->queue_stop) {
 677                 int ret = dev->dev_ops->queue_stop(dev, queue_id);
 678                 if (ret < 0) {
 679                         rte_bbdev_log(ERR, "Device %u queue %u stop failed",
 680                                         dev_id, queue_id);
 681                         return ret;
 682                 }
 683         }
 684         dev->data->queues[queue_id].started = false;
 685
 686         rte_bbdev_log_debug("Stopped queue %u of device %u", queue_id, dev_id);
 687         return 0;
 688 }
 689
 690 /* Get device statistics */
 691 static void
 692 get_stats_from_queues(struct rte_bbdev *dev, struct rte_bbdev_stats *stats)
 693 {
 694         unsigned int q_id;
 695         for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
 696                 struct rte_bbdev_stats *q_stats =
 697                                 &dev->data->queues[q_id].queue_stats;
 698
 699                 stats->enqueued_count += q_stats->enqueued_count;
 700                 stats->dequeued_count += q_stats->dequeued_count;
 701                 stats->enqueue_err_count += q_stats->enqueue_err_count;
 702                 stats->dequeue_err_count += q_stats->dequeue_err_count;
 703         }
 704         rte_bbdev_log_debug("Got stats on %u", dev->data->dev_id);
 705 }
 706
 707 static void
 708 reset_stats_in_queues(struct rte_bbdev *dev)
 709 {
 710         unsigned int q_id;
 711         for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
 712                 struct rte_bbdev_stats *q_stats =
 713                                 &dev->data->queues[q_id].queue_stats;
 714
 715                 memset(q_stats, 0, sizeof(*q_stats));
 716         }
 717         rte_bbdev_log_debug("Reset stats on %u", dev->data->dev_id);
 718 }
 719
 720 int
 721 rte_bbdev_stats_get(uint16_t dev_id, struct rte_bbdev_stats *stats)
 722 {
 723         struct rte_bbdev *dev = get_dev(dev_id);
 724         VALID_DEV_OR_RET_ERR(dev, dev_id);
 725
 726         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 727
 728         if (stats == NULL) {
 729                 rte_bbdev_log(ERR, "NULL stats structure");
 730                 return -EINVAL;
 731         }
 732
 733         memset(stats, 0, sizeof(*stats));
 734         if (dev->dev_ops->stats_get != NULL)
 735                 dev->dev_ops->stats_get(dev, stats);
 736         else
 737                 get_stats_from_queues(dev, stats);
 738
 739         rte_bbdev_log_debug("Retrieved stats of device %u", dev_id);
 740         return 0;
 741 }
 742
 743 int
 744 rte_bbdev_stats_reset(uint16_t dev_id)
 745 {
 746         struct rte_bbdev *dev = get_dev(dev_id);
 747         VALID_DEV_OR_RET_ERR(dev, dev_id);
 748
 749         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 750
 751         if (dev->dev_ops->stats_reset != NULL)
 752                 dev->dev_ops->stats_reset(dev);
 753         else
 754                 reset_stats_in_queues(dev);
 755
 756         rte_bbdev_log_debug("Reset stats of device %u", dev_id);
 757         return 0;
 758 }
 759
 760 int
 761 rte_bbdev_info_get(uint16_t dev_id, struct rte_bbdev_info *dev_info)
 762 {
 763         struct rte_bbdev *dev = get_dev(dev_id);
 764         VALID_DEV_OR_RET_ERR(dev, dev_id);
 765
 766         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 767
 768         if (dev_info == NULL) {
 769                 rte_bbdev_log(ERR, "NULL dev info structure");
 770                 return -EINVAL;
 771         }
 772
 773         /* Copy data maintained by device interface layer */
 774         memset(dev_info, 0, sizeof(*dev_info));
 775         dev_info->dev_name = dev->data->name;
 776         dev_info->num_queues = dev->data->num_queues;
 777         dev_info->bus = rte_bus_find_by_device(dev->device);
 778         dev_info->socket_id = dev->data->socket_id;
 779         dev_info->started = dev->data->started;
 780
 781         /* Copy data maintained by device driver layer */
 782         dev->dev_ops->info_get(dev, &dev_info->drv);
 783
 784         rte_bbdev_log_debug("Retrieved info of device %u", dev_id);
 785         return 0;
 786 }
 787
 788 int
 789 rte_bbdev_queue_info_get(uint16_t dev_id, uint16_t queue_id,
 790                 struct rte_bbdev_queue_info *queue_info)
 791 {
 792         struct rte_bbdev *dev = get_dev(dev_id);
 793         VALID_DEV_OR_RET_ERR(dev, dev_id);
 794
 795         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 796
 797         if (queue_info == NULL) {
 798                 rte_bbdev_log(ERR, "NULL queue info structure");
 799                 return -EINVAL;
 800         }
 801
 802         /* Copy data to output */
 803         memset(queue_info, 0, sizeof(*queue_info));
 804         queue_info->conf = dev->data->queues[queue_id].conf;
 805         queue_info->started = dev->data->queues[queue_id].started;
 806
 807         rte_bbdev_log_debug("Retrieved info of queue %u of device %u",
 808                         queue_id, dev_id);
 809         return 0;
 810 }
 811
 812 /* Calculate size needed to store bbdev_op, depending on type */
 813 static unsigned int
 814 get_bbdev_op_size(enum rte_bbdev_op_type type)
 815 {
 816         unsigned int result = 0;
 817         switch (type) {
 818         case RTE_BBDEV_OP_NONE:
 819                 result = RTE_MAX(sizeof(struct rte_bbdev_dec_op),
 820                                 sizeof(struct rte_bbdev_enc_op));
 821                 break;
 822         case RTE_BBDEV_OP_TURBO_DEC:
 823                 result = sizeof(struct rte_bbdev_dec_op);
 824                 break;
 825         case RTE_BBDEV_OP_TURBO_ENC:
 826                 result = sizeof(struct rte_bbdev_enc_op);
 827                 break;
 828         default:
 829                 break;
 830         }
 831
 832         return result;
 833 }
 834
 835 /* Initialise a bbdev_op structure */
 836 static void
 837 bbdev_op_init(struct rte_mempool *mempool, void *arg, void *element,
 838                 __rte_unused unsigned int n)
 839 {
 840         enum rte_bbdev_op_type type = *(enum rte_bbdev_op_type *)arg;
 841
 842         if (type == RTE_BBDEV_OP_TURBO_DEC) {
 843                 struct rte_bbdev_dec_op *op = element;
 844                 memset(op, 0, mempool->elt_size);
 845                 op->mempool = mempool;
 846         } else if (type == RTE_BBDEV_OP_TURBO_ENC) {
 847                 struct rte_bbdev_enc_op *op = element;
 848                 memset(op, 0, mempool->elt_size);
 849                 op->mempool = mempool;
 850         }
 851 }
 852
 853 struct rte_mempool *
 854 rte_bbdev_op_pool_create(const char *name, enum rte_bbdev_op_type type,
 855                 unsigned int num_elements, unsigned int cache_size,
 856                 int socket_id)
 857 {
 858         struct rte_bbdev_op_pool_private *priv;
 859         struct rte_mempool *mp;
 860         const char *op_type_str;
 861
 862         if (name == NULL) {
 863                 rte_bbdev_log(ERR, "NULL name for op pool");
 864                 return NULL;
 865         }
 866
 867         if (type >= RTE_BBDEV_OP_TYPE_COUNT) {
 868                 rte_bbdev_log(ERR,
 869                                 "Invalid op type (%u), should be less than %u",
 870                                 type, RTE_BBDEV_OP_TYPE_COUNT);
 871                 return NULL;
 872         }
 873
 874         mp = rte_mempool_create(name, num_elements, get_bbdev_op_size(type),
 875                         cache_size, sizeof(struct rte_bbdev_op_pool_private),
 876                         NULL, NULL, bbdev_op_init, &type, socket_id, 0);
 877         if (mp == NULL) {
 878                 rte_bbdev_log(ERR,
 879                                 "Failed to create op pool %s (num ops=%u, op size=%u) with error: %s",
 880                                 name, num_elements, get_bbdev_op_size(type),
 881                                 rte_strerror(rte_errno));
 882                 return NULL;
 883         }
 884
 885         op_type_str = rte_bbdev_op_type_str(type);
 886         if (op_type_str == NULL)
 887                 return NULL;
 888
 889         rte_bbdev_log_debug(
 890                         "Op pool %s created for %u ops (type=%s, cache=%u, socket=%u, size=%u)",
 891                         name, num_elements, op_type_str, cache_size, socket_id,
 892                         get_bbdev_op_size(type));
 893
 894         priv = (struct rte_bbdev_op_pool_private *)rte_mempool_get_priv(mp);
 895         priv->type = type;
 896
 897         return mp;
 898 }
 899
 900 int
 901 rte_bbdev_callback_register(uint16_t dev_id, enum rte_bbdev_event_type event,
 902                 rte_bbdev_cb_fn cb_fn, void *cb_arg)
 903 {
 904         struct rte_bbdev_callback *user_cb;
 905         struct rte_bbdev *dev = get_dev(dev_id);
 906         VALID_DEV_OR_RET_ERR(dev, dev_id);
 907
 908         if (event >= RTE_BBDEV_EVENT_MAX) {
 909                 rte_bbdev_log(ERR,
 910                                 "Invalid event type (%u), should be less than %u",
 911                                 event, RTE_BBDEV_EVENT_MAX);
 912                 return -EINVAL;
 913         }
 914
 915         if (cb_fn == NULL) {
 916                 rte_bbdev_log(ERR, "NULL callback function");
 917                 return -EINVAL;
 918         }
 919
 920         rte_spinlock_lock(&rte_bbdev_cb_lock);
 921
 922         TAILQ_FOREACH(user_cb, &(dev->list_cbs), next) {
 923                 if (user_cb->cb_fn == cb_fn &&
 924                                 user_cb->cb_arg == cb_arg &&
 925                                 user_cb->event == event)
 926                         break;
 927         }
 928
 929         /* create a new callback. */
 930         if (user_cb == NULL) {
 931                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
 932                                 sizeof(struct rte_bbdev_callback), 0);
 933                 if (user_cb != NULL) {
 934                         user_cb->cb_fn = cb_fn;
 935                         user_cb->cb_arg = cb_arg;
 936                         user_cb->event = event;
 937                         TAILQ_INSERT_TAIL(&(dev->list_cbs), user_cb, next);
 938                 }
 939         }
 940
 941         rte_spinlock_unlock(&rte_bbdev_cb_lock);
 942         return (user_cb == NULL) ? -ENOMEM : 0;
 943 }
 944
 945 int
 946 rte_bbdev_callback_unregister(uint16_t dev_id, enum rte_bbdev_event_type event,
 947                 rte_bbdev_cb_fn cb_fn, void *cb_arg)
 948 {
 949         int ret = 0;
 950         struct rte_bbdev_callback *cb, *next;
 951         struct rte_bbdev *dev = get_dev(dev_id);
 952         VALID_DEV_OR_RET_ERR(dev, dev_id);
 953
 954         if (event >= RTE_BBDEV_EVENT_MAX) {
 955                 rte_bbdev_log(ERR,
 956                                 "Invalid event type (%u), should be less than %u",
 957                                 event, RTE_BBDEV_EVENT_MAX);
 958                 return -EINVAL;
 959         }
 960
 961         if (cb_fn == NULL) {
 962                 rte_bbdev_log(ERR,
 963                                 "NULL callback function cannot be unregistered");
 964                 return -EINVAL;
 965         }
 966
 967         dev = &rte_bbdev_devices[dev_id];
 968         rte_spinlock_lock(&rte_bbdev_cb_lock);
 969
 970         for (cb = TAILQ_FIRST(&dev->list_cbs); cb != NULL; cb = next) {
 971
 972                 next = TAILQ_NEXT(cb, next);
 973
 974                 if (cb->cb_fn != cb_fn || cb->event != event ||
 975                                 (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
 976                         continue;
 977
 978                 /* If this callback is not executing right now, remove it. */
 979                 if (cb->active == 0) {
 980                         TAILQ_REMOVE(&(dev->list_cbs), cb, next);
 981                         rte_free(cb);
 982                 } else
 983                         ret = -EAGAIN;
 984         }
 985
 986         rte_spinlock_unlock(&rte_bbdev_cb_lock);
 987         return ret;
 988 }
 989
 990 void
 991 rte_bbdev_pmd_callback_process(struct rte_bbdev *dev,
 992         enum rte_bbdev_event_type event, void *ret_param)
 993 {
 994         struct rte_bbdev_callback *cb_lst;
 995         struct rte_bbdev_callback dev_cb;
 996
 997         if (dev == NULL) {
 998                 rte_bbdev_log(ERR, "NULL device");
 999                 return;
1000         }
1001
1002         if (dev->data == NULL) {
1003                 rte_bbdev_log(ERR, "NULL data structure");
1004                 return;
1005         }
1006
1007         if (event >= RTE_BBDEV_EVENT_MAX) {
1008                 rte_bbdev_log(ERR,
1009                                 "Invalid event type (%u), should be less than %u",
1010                                 event, RTE_BBDEV_EVENT_MAX);
1011                 return;
1012         }
1013
1014         rte_spinlock_lock(&rte_bbdev_cb_lock);
1015         TAILQ_FOREACH(cb_lst, &(dev->list_cbs), next) {
1016                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1017                         continue;
1018                 dev_cb = *cb_lst;
1019                 cb_lst->active = 1;
1020                 if (ret_param != NULL)
1021                         dev_cb.ret_param = ret_param;
1022
1023                 rte_spinlock_unlock(&rte_bbdev_cb_lock);
1024                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1025                                 dev_cb.cb_arg, dev_cb.ret_param);
1026                 rte_spinlock_lock(&rte_bbdev_cb_lock);
1027                 cb_lst->active = 0;
1028         }
1029         rte_spinlock_unlock(&rte_bbdev_cb_lock);
1030 }
1031
1032 int
1033 rte_bbdev_queue_intr_enable(uint16_t dev_id, uint16_t queue_id)
1034 {
1035         struct rte_bbdev *dev = get_dev(dev_id);
1036         VALID_DEV_OR_RET_ERR(dev, dev_id);
1037         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1038         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1039         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_enable, dev_id);
1040         return dev->dev_ops->queue_intr_enable(dev, queue_id);
1041 }
1042
1043 int
1044 rte_bbdev_queue_intr_disable(uint16_t dev_id, uint16_t queue_id)
1045 {
1046         struct rte_bbdev *dev = get_dev(dev_id);
1047         VALID_DEV_OR_RET_ERR(dev, dev_id);
1048         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1049         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1050         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_disable, dev_id);
1051         return dev->dev_ops->queue_intr_disable(dev, queue_id);
1052 }
1053
1054 int
1055 rte_bbdev_queue_intr_ctl(uint16_t dev_id, uint16_t queue_id, int epfd, int op,
1056                 void *data)
1057 {
1058         uint32_t vec;
1059         struct rte_bbdev *dev = get_dev(dev_id);
1060         struct rte_intr_handle *intr_handle;
1061         int ret;
1062
1063         VALID_DEV_OR_RET_ERR(dev, dev_id);
1064         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1065
1066         intr_handle = dev->intr_handle;
1067         if (!intr_handle || !intr_handle->intr_vec) {
1068                 rte_bbdev_log(ERR, "Device %u intr handle unset\n", dev_id);
1069                 return -ENOTSUP;
1070         }
1071
1072         if (queue_id >= RTE_MAX_RXTX_INTR_VEC_ID) {
1073                 rte_bbdev_log(ERR, "Device %u queue_id %u is too big\n",
1074                                 dev_id, queue_id);
1075                 return -ENOTSUP;
1076         }
1077
1078         vec = intr_handle->intr_vec[queue_id];
1079         ret = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
1080         if (ret && (ret != -EEXIST)) {
1081                 rte_bbdev_log(ERR,
1082                                 "dev %u q %u int ctl error op %d epfd %d vec %u\n",
1083                                 dev_id, queue_id, op, epfd, vec);
1084                 return ret;
1085         }
1086
1087         return 0;
1088 }
1089
1090
1091 const char *
1092 rte_bbdev_op_type_str(enum rte_bbdev_op_type op_type)
1093 {
1094         static const char * const op_types[] = {
1095                 "RTE_BBDEV_OP_NONE",
1096                 "RTE_BBDEV_OP_TURBO_DEC",
1097                 "RTE_BBDEV_OP_TURBO_ENC",
1098         };
1099
1100         if (op_type < RTE_BBDEV_OP_TYPE_COUNT)
1101                 return op_types[op_type];
1102
1103         rte_bbdev_log(ERR, "Invalid operation type");
1104         return NULL;
1105 }
1106
1107
1108 int bbdev_logtype;
1109
1110 RTE_INIT(rte_bbdev_init_log);
1111 static void
1112 rte_bbdev_init_log(void)
1113 {
1114         bbdev_logtype = rte_log_register("lib.bbdev");
1115         if (bbdev_logtype >= 0)
1116                 rte_log_set_level(bbdev_logtype, RTE_LOG_NOTICE);
1117 }