lib/bbdev/rte_bbdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_common.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12 #include <rte_eal.h>
  13 #include <rte_malloc.h>
  14 #include <rte_mempool.h>
  15 #include <rte_memzone.h>
  16 #include <rte_lcore.h>
  17 #include <rte_spinlock.h>
  18 #include <rte_interrupts.h>
  19
  20 #include "rte_bbdev_op.h"
  21 #include "rte_bbdev.h"
  22 #include "rte_bbdev_pmd.h"
  23
  24 #define DEV_NAME "BBDEV"
  25
  26
  27 /* BBDev library logging ID */
  28 RTE_LOG_REGISTER_DEFAULT(bbdev_logtype, NOTICE);
  29
  30 /* Helper macro for logging */
  31 #define rte_bbdev_log(level, fmt, ...) \
  32         rte_log(RTE_LOG_ ## level, bbdev_logtype, fmt "\n", ##__VA_ARGS__)
  33
  34 #define rte_bbdev_log_debug(fmt, ...) \
  35         rte_bbdev_log(DEBUG, RTE_STR(__LINE__) ":%s() " fmt, __func__, \
  36                 ##__VA_ARGS__)
  37
  38 /* Helper macro to check dev_id is valid */
  39 #define VALID_DEV_OR_RET_ERR(dev, dev_id) do { \
  40         if (dev == NULL) { \
  41                 rte_bbdev_log(ERR, "device %u is invalid", dev_id); \
  42                 return -ENODEV; \
  43         } \
  44 } while (0)
  45
  46 /* Helper macro to check dev_ops is valid */
  47 #define VALID_DEV_OPS_OR_RET_ERR(dev, dev_id) do { \
  48         if (dev->dev_ops == NULL) { \
  49                 rte_bbdev_log(ERR, "NULL dev_ops structure in device %u", \
  50                                 dev_id); \
  51                 return -ENODEV; \
  52         } \
  53 } while (0)
  54
  55 /* Helper macro to check that driver implements required function pointer */
  56 #define VALID_FUNC_OR_RET_ERR(func, dev_id) do { \
  57         if (func == NULL) { \
  58                 rte_bbdev_log(ERR, "device %u does not support %s", \
  59                                 dev_id, #func); \
  60                 return -ENOTSUP; \
  61         } \
  62 } while (0)
  63
  64 /* Helper macro to check that queue is valid */
  65 #define VALID_QUEUE_OR_RET_ERR(queue_id, dev) do { \
  66         if (queue_id >= dev->data->num_queues) { \
  67                 rte_bbdev_log(ERR, "Invalid queue_id %u for device %u", \
  68                                 queue_id, dev->data->dev_id); \
  69                 return -ERANGE; \
  70         } \
  71 } while (0)
  72
  73 /* List of callback functions registered by an application */
  74 struct rte_bbdev_callback {
  75         TAILQ_ENTRY(rte_bbdev_callback) next;  /* Callbacks list */
  76         rte_bbdev_cb_fn cb_fn;  /* Callback address */
  77         void *cb_arg;  /* Parameter for callback */
  78         void *ret_param;  /* Return parameter */
  79         enum rte_bbdev_event_type event; /* Interrupt event type */
  80         uint32_t active; /* Callback is executing */
  81 };
  82
  83 /* spinlock for bbdev device callbacks */
  84 static rte_spinlock_t rte_bbdev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  85
  86 /*
  87  * Global array of all devices. This is not static because it's used by the
  88  * inline enqueue and dequeue functions
  89  */
  90 struct rte_bbdev rte_bbdev_devices[RTE_BBDEV_MAX_DEVS];
  91
  92 /* Global array with rte_bbdev_data structures */
  93 static struct rte_bbdev_data *rte_bbdev_data;
  94
  95 /* Memzone name for global bbdev data pool */
  96 static const char *MZ_RTE_BBDEV_DATA = "rte_bbdev_data";
  97
  98 /* Number of currently valid devices */
  99 static uint16_t num_devs;
 100
 101 /* Return pointer to device structure, with validity check */
 102 static struct rte_bbdev *
 103 get_dev(uint16_t dev_id)
 104 {
 105         if (rte_bbdev_is_valid(dev_id))
 106                 return &rte_bbdev_devices[dev_id];
 107         return NULL;
 108 }
 109
 110 /* Allocate global data array */
 111 static int
 112 rte_bbdev_data_alloc(void)
 113 {
 114         const unsigned int flags = 0;
 115         const struct rte_memzone *mz;
 116
 117         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 118                 mz = rte_memzone_reserve(MZ_RTE_BBDEV_DATA,
 119                                 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data),
 120                                 rte_socket_id(), flags);
 121         } else
 122                 mz = rte_memzone_lookup(MZ_RTE_BBDEV_DATA);
 123         if (mz == NULL) {
 124                 rte_bbdev_log(CRIT,
 125                                 "Cannot allocate memzone for bbdev port data");
 126                 return -ENOMEM;
 127         }
 128
 129         rte_bbdev_data = mz->addr;
 130         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 131                 memset(rte_bbdev_data, 0,
 132                                 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data));
 133         return 0;
 134 }
 135
 136 /*
 137  * Find data allocated for the device or if not found return first unused bbdev
 138  * data. If all structures are in use and none is used by the device return
 139  * NULL.
 140  */
 141 static struct rte_bbdev_data *
 142 find_bbdev_data(const char *name)
 143 {
 144         uint16_t data_id;
 145
 146         for (data_id = 0; data_id < RTE_BBDEV_MAX_DEVS; ++data_id) {
 147                 if (strlen(rte_bbdev_data[data_id].name) == 0) {
 148                         memset(&rte_bbdev_data[data_id], 0,
 149                                         sizeof(struct rte_bbdev_data));
 150                         return &rte_bbdev_data[data_id];
 151                 } else if (strncmp(rte_bbdev_data[data_id].name, name,
 152                                 RTE_BBDEV_NAME_MAX_LEN) == 0)
 153                         return &rte_bbdev_data[data_id];
 154         }
 155
 156         return NULL;
 157 }
 158
 159 /* Find lowest device id with no attached device */
 160 static uint16_t
 161 find_free_dev_id(void)
 162 {
 163         uint16_t i;
 164         for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
 165                 if (rte_bbdev_devices[i].state == RTE_BBDEV_UNUSED)
 166                         return i;
 167         }
 168         return RTE_BBDEV_MAX_DEVS;
 169 }
 170
 171 struct rte_bbdev *
 172 rte_bbdev_allocate(const char *name)
 173 {
 174         int ret;
 175         struct rte_bbdev *bbdev;
 176         uint16_t dev_id;
 177
 178         if (name == NULL) {
 179                 rte_bbdev_log(ERR, "Invalid null device name");
 180                 return NULL;
 181         }
 182
 183         if (rte_bbdev_get_named_dev(name) != NULL) {
 184                 rte_bbdev_log(ERR, "Device \"%s\" is already allocated", name);
 185                 return NULL;
 186         }
 187
 188         dev_id = find_free_dev_id();
 189         if (dev_id == RTE_BBDEV_MAX_DEVS) {
 190                 rte_bbdev_log(ERR, "Reached maximum number of devices");
 191                 return NULL;
 192         }
 193
 194         bbdev = &rte_bbdev_devices[dev_id];
 195
 196         if (rte_bbdev_data == NULL) {
 197                 ret = rte_bbdev_data_alloc();
 198                 if (ret != 0)
 199                         return NULL;
 200         }
 201
 202         bbdev->data = find_bbdev_data(name);
 203         if (bbdev->data == NULL) {
 204                 rte_bbdev_log(ERR,
 205                                 "Max BBDevs already allocated in multi-process environment!");
 206                 return NULL;
 207         }
 208
 209         __atomic_add_fetch(&bbdev->data->process_cnt, 1, __ATOMIC_RELAXED);
 210         bbdev->data->dev_id = dev_id;
 211         bbdev->state = RTE_BBDEV_INITIALIZED;
 212
 213         ret = snprintf(bbdev->data->name, RTE_BBDEV_NAME_MAX_LEN, "%s", name);
 214         if ((ret < 0) || (ret >= RTE_BBDEV_NAME_MAX_LEN)) {
 215                 rte_bbdev_log(ERR, "Copying device name \"%s\" failed", name);
 216                 return NULL;
 217         }
 218
 219         /* init user callbacks */
 220         TAILQ_INIT(&(bbdev->list_cbs));
 221
 222         num_devs++;
 223
 224         rte_bbdev_log_debug("Initialised device %s (id = %u). Num devices = %u",
 225                         name, dev_id, num_devs);
 226
 227         return bbdev;
 228 }
 229
 230 int
 231 rte_bbdev_release(struct rte_bbdev *bbdev)
 232 {
 233         uint16_t dev_id;
 234         struct rte_bbdev_callback *cb, *next;
 235
 236         if (bbdev == NULL) {
 237                 rte_bbdev_log(ERR, "NULL bbdev");
 238                 return -ENODEV;
 239         }
 240         dev_id = bbdev->data->dev_id;
 241
 242         /* free all callbacks from the device's list */
 243         for (cb = TAILQ_FIRST(&bbdev->list_cbs); cb != NULL; cb = next) {
 244
 245                 next = TAILQ_NEXT(cb, next);
 246                 TAILQ_REMOVE(&(bbdev->list_cbs), cb, next);
 247                 rte_free(cb);
 248         }
 249
 250         /* clear shared BBDev Data if no process is using the device anymore */
 251         if (__atomic_sub_fetch(&bbdev->data->process_cnt, 1,
 252                               __ATOMIC_RELAXED) == 0)
 253                 memset(bbdev->data, 0, sizeof(*bbdev->data));
 254
 255         memset(bbdev, 0, sizeof(*bbdev));
 256         num_devs--;
 257         bbdev->state = RTE_BBDEV_UNUSED;
 258
 259         rte_bbdev_log_debug(
 260                         "Un-initialised device id = %u. Num devices = %u",
 261                         dev_id, num_devs);
 262         return 0;
 263 }
 264
 265 struct rte_bbdev *
 266 rte_bbdev_get_named_dev(const char *name)
 267 {
 268         unsigned int i;
 269
 270         if (name == NULL) {
 271                 rte_bbdev_log(ERR, "NULL driver name");
 272                 return NULL;
 273         }
 274
 275         for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
 276                 struct rte_bbdev *dev = get_dev(i);
 277                 if (dev && (strncmp(dev->data->name,
 278                                 name, RTE_BBDEV_NAME_MAX_LEN) == 0))
 279                         return dev;
 280         }
 281
 282         return NULL;
 283 }
 284
 285 uint16_t
 286 rte_bbdev_count(void)
 287 {
 288         return num_devs;
 289 }
 290
 291 bool
 292 rte_bbdev_is_valid(uint16_t dev_id)
 293 {
 294         if ((dev_id < RTE_BBDEV_MAX_DEVS) &&
 295                 rte_bbdev_devices[dev_id].state == RTE_BBDEV_INITIALIZED)
 296                 return true;
 297         return false;
 298 }
 299
 300 uint16_t
 301 rte_bbdev_find_next(uint16_t dev_id)
 302 {
 303         dev_id++;
 304         for (; dev_id < RTE_BBDEV_MAX_DEVS; dev_id++)
 305                 if (rte_bbdev_is_valid(dev_id))
 306                         break;
 307         return dev_id;
 308 }
 309
 310 int
 311 rte_bbdev_setup_queues(uint16_t dev_id, uint16_t num_queues, int socket_id)
 312 {
 313         unsigned int i;
 314         int ret;
 315         struct rte_bbdev_driver_info dev_info;
 316         struct rte_bbdev *dev = get_dev(dev_id);
 317         VALID_DEV_OR_RET_ERR(dev, dev_id);
 318
 319         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 320
 321         if (dev->data->started) {
 322                 rte_bbdev_log(ERR,
 323                                 "Device %u cannot be configured when started",
 324                                 dev_id);
 325                 return -EBUSY;
 326         }
 327
 328         /* Get device driver information to get max number of queues */
 329         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 330         memset(&dev_info, 0, sizeof(dev_info));
 331         dev->dev_ops->info_get(dev, &dev_info);
 332
 333         if ((num_queues == 0) || (num_queues > dev_info.max_num_queues)) {
 334                 rte_bbdev_log(ERR,
 335                                 "Device %u supports 0 < N <= %u queues, not %u",
 336                                 dev_id, dev_info.max_num_queues, num_queues);
 337                 return -EINVAL;
 338         }
 339
 340         /* If re-configuration, get driver to free existing internal memory */
 341         if (dev->data->queues != NULL) {
 342                 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
 343                 for (i = 0; i < dev->data->num_queues; i++) {
 344                         int ret = dev->dev_ops->queue_release(dev, i);
 345                         if (ret < 0) {
 346                                 rte_bbdev_log(ERR,
 347                                                 "Device %u queue %u release failed",
 348                                                 dev_id, i);
 349                                 return ret;
 350                         }
 351                 }
 352                 /* Call optional device close */
 353                 if (dev->dev_ops->close) {
 354                         ret = dev->dev_ops->close(dev);
 355                         if (ret < 0) {
 356                                 rte_bbdev_log(ERR,
 357                                                 "Device %u couldn't be closed",
 358                                                 dev_id);
 359                                 return ret;
 360                         }
 361                 }
 362                 rte_free(dev->data->queues);
 363         }
 364
 365         /* Allocate queue pointers */
 366         dev->data->queues = rte_calloc_socket(DEV_NAME, num_queues,
 367                         sizeof(dev->data->queues[0]), RTE_CACHE_LINE_SIZE,
 368                                 dev->data->socket_id);
 369         if (dev->data->queues == NULL) {
 370                 rte_bbdev_log(ERR,
 371                                 "calloc of %u queues for device %u on socket %i failed",
 372                                 num_queues, dev_id, dev->data->socket_id);
 373                 return -ENOMEM;
 374         }
 375
 376         dev->data->num_queues = num_queues;
 377
 378         /* Call optional device configuration */
 379         if (dev->dev_ops->setup_queues) {
 380                 ret = dev->dev_ops->setup_queues(dev, num_queues, socket_id);
 381                 if (ret < 0) {
 382                         rte_bbdev_log(ERR,
 383                                         "Device %u memory configuration failed",
 384                                         dev_id);
 385                         goto error;
 386                 }
 387         }
 388
 389         rte_bbdev_log_debug("Device %u set up with %u queues", dev_id,
 390                         num_queues);
 391         return 0;
 392
 393 error:
 394         dev->data->num_queues = 0;
 395         rte_free(dev->data->queues);
 396         dev->data->queues = NULL;
 397         return ret;
 398 }
 399
 400 int
 401 rte_bbdev_intr_enable(uint16_t dev_id)
 402 {
 403         int ret;
 404         struct rte_bbdev *dev = get_dev(dev_id);
 405         VALID_DEV_OR_RET_ERR(dev, dev_id);
 406
 407         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 408
 409         if (dev->data->started) {
 410                 rte_bbdev_log(ERR,
 411                                 "Device %u cannot be configured when started",
 412                                 dev_id);
 413                 return -EBUSY;
 414         }
 415
 416         if (dev->dev_ops->intr_enable) {
 417                 ret = dev->dev_ops->intr_enable(dev);
 418                 if (ret < 0) {
 419                         rte_bbdev_log(ERR,
 420                                         "Device %u interrupts configuration failed",
 421                                         dev_id);
 422                         return ret;
 423                 }
 424                 rte_bbdev_log_debug("Enabled interrupts for dev %u", dev_id);
 425                 return 0;
 426         }
 427
 428         rte_bbdev_log(ERR, "Device %u doesn't support interrupts", dev_id);
 429         return -ENOTSUP;
 430 }
 431
 432 int
 433 rte_bbdev_queue_configure(uint16_t dev_id, uint16_t queue_id,
 434                 const struct rte_bbdev_queue_conf *conf)
 435 {
 436         int ret = 0;
 437         struct rte_bbdev_driver_info dev_info;
 438         struct rte_bbdev *dev = get_dev(dev_id);
 439         const struct rte_bbdev_op_cap *p;
 440         struct rte_bbdev_queue_conf *stored_conf;
 441         const char *op_type_str;
 442         VALID_DEV_OR_RET_ERR(dev, dev_id);
 443
 444         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 445
 446         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 447
 448         if (dev->data->queues[queue_id].started || dev->data->started) {
 449                 rte_bbdev_log(ERR,
 450                                 "Queue %u of device %u cannot be configured when started",
 451                                 queue_id, dev_id);
 452                 return -EBUSY;
 453         }
 454
 455         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
 456         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_setup, dev_id);
 457
 458         /* Get device driver information to verify config is valid */
 459         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 460         memset(&dev_info, 0, sizeof(dev_info));
 461         dev->dev_ops->info_get(dev, &dev_info);
 462
 463         /* Check configuration is valid */
 464         if (conf != NULL) {
 465                 if ((conf->op_type == RTE_BBDEV_OP_NONE) &&
 466                                 (dev_info.capabilities[0].type ==
 467                                 RTE_BBDEV_OP_NONE)) {
 468                         ret = 1;
 469                 } else {
 470                         for (p = dev_info.capabilities;
 471                                         p->type != RTE_BBDEV_OP_NONE; p++) {
 472                                 if (conf->op_type == p->type) {
 473                                         ret = 1;
 474                                         break;
 475                                 }
 476                         }
 477                 }
 478                 if (ret == 0) {
 479                         rte_bbdev_log(ERR, "Invalid operation type");
 480                         return -EINVAL;
 481                 }
 482                 if (conf->queue_size > dev_info.queue_size_lim) {
 483                         rte_bbdev_log(ERR,
 484                                         "Size (%u) of queue %u of device %u must be: <= %u",
 485                                         conf->queue_size, queue_id, dev_id,
 486                                         dev_info.queue_size_lim);
 487                         return -EINVAL;
 488                 }
 489                 if (!rte_is_power_of_2(conf->queue_size)) {
 490                         rte_bbdev_log(ERR,
 491                                         "Size (%u) of queue %u of device %u must be a power of 2",
 492                                         conf->queue_size, queue_id, dev_id);
 493                         return -EINVAL;
 494                 }
 495                 if (conf->op_type == RTE_BBDEV_OP_TURBO_DEC &&
 496                         conf->priority > dev_info.max_ul_queue_priority) {
 497                         rte_bbdev_log(ERR,
 498                                         "Priority (%u) of queue %u of bbdev %u must be <= %u",
 499                                         conf->priority, queue_id, dev_id,
 500                                         dev_info.max_ul_queue_priority);
 501                         return -EINVAL;
 502                 }
 503                 if (conf->op_type == RTE_BBDEV_OP_TURBO_ENC &&
 504                         conf->priority > dev_info.max_dl_queue_priority) {
 505                         rte_bbdev_log(ERR,
 506                                         "Priority (%u) of queue %u of bbdev %u must be <= %u",
 507                                         conf->priority, queue_id, dev_id,
 508                                         dev_info.max_dl_queue_priority);
 509                         return -EINVAL;
 510                 }
 511         }
 512
 513         /* Release existing queue (in case of queue reconfiguration) */
 514         if (dev->data->queues[queue_id].queue_private != NULL) {
 515                 ret = dev->dev_ops->queue_release(dev, queue_id);
 516                 if (ret < 0) {
 517                         rte_bbdev_log(ERR, "Device %u queue %u release failed",
 518                                         dev_id, queue_id);
 519                         return ret;
 520                 }
 521         }
 522
 523         /* Get driver to setup the queue */
 524         ret = dev->dev_ops->queue_setup(dev, queue_id, (conf != NULL) ?
 525                         conf : &dev_info.default_queue_conf);
 526         if (ret < 0) {
 527                 /* This may happen when trying different priority levels */
 528                 rte_bbdev_log(INFO,
 529                                 "Device %u queue %u setup failed",
 530                                 dev_id, queue_id);
 531                 return ret;
 532         }
 533
 534         /* Store configuration */
 535         stored_conf = &dev->data->queues[queue_id].conf;
 536         memcpy(stored_conf,
 537                         (conf != NULL) ? conf : &dev_info.default_queue_conf,
 538                         sizeof(*stored_conf));
 539
 540         op_type_str = rte_bbdev_op_type_str(stored_conf->op_type);
 541         if (op_type_str == NULL)
 542                 return -EINVAL;
 543
 544         rte_bbdev_log_debug("Configured dev%uq%u (size=%u, type=%s, prio=%u)",
 545                         dev_id, queue_id, stored_conf->queue_size, op_type_str,
 546                         stored_conf->priority);
 547
 548         return 0;
 549 }
 550
 551 int
 552 rte_bbdev_start(uint16_t dev_id)
 553 {
 554         int i;
 555         struct rte_bbdev *dev = get_dev(dev_id);
 556         VALID_DEV_OR_RET_ERR(dev, dev_id);
 557
 558         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 559
 560         if (dev->data->started) {
 561                 rte_bbdev_log_debug("Device %u is already started", dev_id);
 562                 return 0;
 563         }
 564
 565         if (dev->dev_ops->start) {
 566                 int ret = dev->dev_ops->start(dev);
 567                 if (ret < 0) {
 568                         rte_bbdev_log(ERR, "Device %u start failed", dev_id);
 569                         return ret;
 570                 }
 571         }
 572
 573         /* Store new state */
 574         for (i = 0; i < dev->data->num_queues; i++)
 575                 if (!dev->data->queues[i].conf.deferred_start)
 576                         dev->data->queues[i].started = true;
 577         dev->data->started = true;
 578
 579         rte_bbdev_log_debug("Started device %u", dev_id);
 580         return 0;
 581 }
 582
 583 int
 584 rte_bbdev_stop(uint16_t dev_id)
 585 {
 586         struct rte_bbdev *dev = get_dev(dev_id);
 587         VALID_DEV_OR_RET_ERR(dev, dev_id);
 588
 589         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 590
 591         if (!dev->data->started) {
 592                 rte_bbdev_log_debug("Device %u is already stopped", dev_id);
 593                 return 0;
 594         }
 595
 596         if (dev->dev_ops->stop)
 597                 dev->dev_ops->stop(dev);
 598         dev->data->started = false;
 599
 600         rte_bbdev_log_debug("Stopped device %u", dev_id);
 601         return 0;
 602 }
 603
 604 int
 605 rte_bbdev_close(uint16_t dev_id)
 606 {
 607         int ret;
 608         uint16_t i;
 609         struct rte_bbdev *dev = get_dev(dev_id);
 610         VALID_DEV_OR_RET_ERR(dev, dev_id);
 611
 612         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 613
 614         if (dev->data->started) {
 615                 ret = rte_bbdev_stop(dev_id);
 616                 if (ret < 0) {
 617                         rte_bbdev_log(ERR, "Device %u stop failed", dev_id);
 618                         return ret;
 619                 }
 620         }
 621
 622         /* Free memory used by queues */
 623         for (i = 0; i < dev->data->num_queues; i++) {
 624                 ret = dev->dev_ops->queue_release(dev, i);
 625                 if (ret < 0) {
 626                         rte_bbdev_log(ERR, "Device %u queue %u release failed",
 627                                         dev_id, i);
 628                         return ret;
 629                 }
 630         }
 631         rte_free(dev->data->queues);
 632
 633         if (dev->dev_ops->close) {
 634                 ret = dev->dev_ops->close(dev);
 635                 if (ret < 0) {
 636                         rte_bbdev_log(ERR, "Device %u close failed", dev_id);
 637                         return ret;
 638                 }
 639         }
 640
 641         /* Clear configuration */
 642         dev->data->queues = NULL;
 643         dev->data->num_queues = 0;
 644
 645         rte_bbdev_log_debug("Closed device %u", dev_id);
 646         return 0;
 647 }
 648
 649 int
 650 rte_bbdev_queue_start(uint16_t dev_id, uint16_t queue_id)
 651 {
 652         struct rte_bbdev *dev = get_dev(dev_id);
 653         VALID_DEV_OR_RET_ERR(dev, dev_id);
 654
 655         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 656
 657         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 658
 659         if (dev->data->queues[queue_id].started) {
 660                 rte_bbdev_log_debug("Queue %u of device %u already started",
 661                                 queue_id, dev_id);
 662                 return 0;
 663         }
 664
 665         if (dev->dev_ops->queue_start) {
 666                 int ret = dev->dev_ops->queue_start(dev, queue_id);
 667                 if (ret < 0) {
 668                         rte_bbdev_log(ERR, "Device %u queue %u start failed",
 669                                         dev_id, queue_id);
 670                         return ret;
 671                 }
 672         }
 673         dev->data->queues[queue_id].started = true;
 674
 675         rte_bbdev_log_debug("Started queue %u of device %u", queue_id, dev_id);
 676         return 0;
 677 }
 678
 679 int
 680 rte_bbdev_queue_stop(uint16_t dev_id, uint16_t queue_id)
 681 {
 682         struct rte_bbdev *dev = get_dev(dev_id);
 683         VALID_DEV_OR_RET_ERR(dev, dev_id);
 684
 685         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 686
 687         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 688
 689         if (!dev->data->queues[queue_id].started) {
 690                 rte_bbdev_log_debug("Queue %u of device %u already stopped",
 691                                 queue_id, dev_id);
 692                 return 0;
 693         }
 694
 695         if (dev->dev_ops->queue_stop) {
 696                 int ret = dev->dev_ops->queue_stop(dev, queue_id);
 697                 if (ret < 0) {
 698                         rte_bbdev_log(ERR, "Device %u queue %u stop failed",
 699                                         dev_id, queue_id);
 700                         return ret;
 701                 }
 702         }
 703         dev->data->queues[queue_id].started = false;
 704
 705         rte_bbdev_log_debug("Stopped queue %u of device %u", queue_id, dev_id);
 706         return 0;
 707 }
 708
 709 /* Get device statistics */
 710 static void
 711 get_stats_from_queues(struct rte_bbdev *dev, struct rte_bbdev_stats *stats)
 712 {
 713         unsigned int q_id;
 714         for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
 715                 struct rte_bbdev_stats *q_stats =
 716                                 &dev->data->queues[q_id].queue_stats;
 717
 718                 stats->enqueued_count += q_stats->enqueued_count;
 719                 stats->dequeued_count += q_stats->dequeued_count;
 720                 stats->enqueue_err_count += q_stats->enqueue_err_count;
 721                 stats->dequeue_err_count += q_stats->dequeue_err_count;
 722         }
 723         rte_bbdev_log_debug("Got stats on %u", dev->data->dev_id);
 724 }
 725
 726 static void
 727 reset_stats_in_queues(struct rte_bbdev *dev)
 728 {
 729         unsigned int q_id;
 730         for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
 731                 struct rte_bbdev_stats *q_stats =
 732                                 &dev->data->queues[q_id].queue_stats;
 733
 734                 memset(q_stats, 0, sizeof(*q_stats));
 735         }
 736         rte_bbdev_log_debug("Reset stats on %u", dev->data->dev_id);
 737 }
 738
 739 int
 740 rte_bbdev_stats_get(uint16_t dev_id, struct rte_bbdev_stats *stats)
 741 {
 742         struct rte_bbdev *dev = get_dev(dev_id);
 743         VALID_DEV_OR_RET_ERR(dev, dev_id);
 744
 745         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 746
 747         if (stats == NULL) {
 748                 rte_bbdev_log(ERR, "NULL stats structure");
 749                 return -EINVAL;
 750         }
 751
 752         memset(stats, 0, sizeof(*stats));
 753         if (dev->dev_ops->stats_get != NULL)
 754                 dev->dev_ops->stats_get(dev, stats);
 755         else
 756                 get_stats_from_queues(dev, stats);
 757
 758         rte_bbdev_log_debug("Retrieved stats of device %u", dev_id);
 759         return 0;
 760 }
 761
 762 int
 763 rte_bbdev_stats_reset(uint16_t dev_id)
 764 {
 765         struct rte_bbdev *dev = get_dev(dev_id);
 766         VALID_DEV_OR_RET_ERR(dev, dev_id);
 767
 768         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 769
 770         if (dev->dev_ops->stats_reset != NULL)
 771                 dev->dev_ops->stats_reset(dev);
 772         else
 773                 reset_stats_in_queues(dev);
 774
 775         rte_bbdev_log_debug("Reset stats of device %u", dev_id);
 776         return 0;
 777 }
 778
 779 int
 780 rte_bbdev_info_get(uint16_t dev_id, struct rte_bbdev_info *dev_info)
 781 {
 782         struct rte_bbdev *dev = get_dev(dev_id);
 783         VALID_DEV_OR_RET_ERR(dev, dev_id);
 784
 785         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 786
 787         if (dev_info == NULL) {
 788                 rte_bbdev_log(ERR, "NULL dev info structure");
 789                 return -EINVAL;
 790         }
 791
 792         /* Copy data maintained by device interface layer */
 793         memset(dev_info, 0, sizeof(*dev_info));
 794         dev_info->dev_name = dev->data->name;
 795         dev_info->num_queues = dev->data->num_queues;
 796         dev_info->device = dev->device;
 797         dev_info->socket_id = dev->data->socket_id;
 798         dev_info->started = dev->data->started;
 799
 800         /* Copy data maintained by device driver layer */
 801         dev->dev_ops->info_get(dev, &dev_info->drv);
 802
 803         rte_bbdev_log_debug("Retrieved info of device %u", dev_id);
 804         return 0;
 805 }
 806
 807 int
 808 rte_bbdev_queue_info_get(uint16_t dev_id, uint16_t queue_id,
 809                 struct rte_bbdev_queue_info *queue_info)
 810 {
 811         struct rte_bbdev *dev = get_dev(dev_id);
 812         VALID_DEV_OR_RET_ERR(dev, dev_id);
 813
 814         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 815
 816         if (queue_info == NULL) {
 817                 rte_bbdev_log(ERR, "NULL queue info structure");
 818                 return -EINVAL;
 819         }
 820
 821         /* Copy data to output */
 822         memset(queue_info, 0, sizeof(*queue_info));
 823         queue_info->conf = dev->data->queues[queue_id].conf;
 824         queue_info->started = dev->data->queues[queue_id].started;
 825
 826         rte_bbdev_log_debug("Retrieved info of queue %u of device %u",
 827                         queue_id, dev_id);
 828         return 0;
 829 }
 830
 831 /* Calculate size needed to store bbdev_op, depending on type */
 832 static unsigned int
 833 get_bbdev_op_size(enum rte_bbdev_op_type type)
 834 {
 835         unsigned int result = 0;
 836         switch (type) {
 837         case RTE_BBDEV_OP_NONE:
 838                 result = RTE_MAX(sizeof(struct rte_bbdev_dec_op),
 839                                 sizeof(struct rte_bbdev_enc_op));
 840                 break;
 841         case RTE_BBDEV_OP_TURBO_DEC:
 842                 result = sizeof(struct rte_bbdev_dec_op);
 843                 break;
 844         case RTE_BBDEV_OP_TURBO_ENC:
 845                 result = sizeof(struct rte_bbdev_enc_op);
 846                 break;
 847         case RTE_BBDEV_OP_LDPC_DEC:
 848                 result = sizeof(struct rte_bbdev_dec_op);
 849                 break;
 850         case RTE_BBDEV_OP_LDPC_ENC:
 851                 result = sizeof(struct rte_bbdev_enc_op);
 852                 break;
 853         default:
 854                 break;
 855         }
 856
 857         return result;
 858 }
 859
 860 /* Initialise a bbdev_op structure */
 861 static void
 862 bbdev_op_init(struct rte_mempool *mempool, void *arg, void *element,
 863                 __rte_unused unsigned int n)
 864 {
 865         enum rte_bbdev_op_type type = *(enum rte_bbdev_op_type *)arg;
 866
 867         if (type == RTE_BBDEV_OP_TURBO_DEC || type == RTE_BBDEV_OP_LDPC_DEC) {
 868                 struct rte_bbdev_dec_op *op = element;
 869                 memset(op, 0, mempool->elt_size);
 870                 op->mempool = mempool;
 871         } else if (type == RTE_BBDEV_OP_TURBO_ENC ||
 872                         type == RTE_BBDEV_OP_LDPC_ENC) {
 873                 struct rte_bbdev_enc_op *op = element;
 874                 memset(op, 0, mempool->elt_size);
 875                 op->mempool = mempool;
 876         }
 877 }
 878
 879 struct rte_mempool *
 880 rte_bbdev_op_pool_create(const char *name, enum rte_bbdev_op_type type,
 881                 unsigned int num_elements, unsigned int cache_size,
 882                 int socket_id)
 883 {
 884         struct rte_bbdev_op_pool_private *priv;
 885         struct rte_mempool *mp;
 886         const char *op_type_str;
 887
 888         if (name == NULL) {
 889                 rte_bbdev_log(ERR, "NULL name for op pool");
 890                 return NULL;
 891         }
 892
 893         if (type >= RTE_BBDEV_OP_TYPE_COUNT) {
 894                 rte_bbdev_log(ERR,
 895                                 "Invalid op type (%u), should be less than %u",
 896                                 type, RTE_BBDEV_OP_TYPE_COUNT);
 897                 return NULL;
 898         }
 899
 900         mp = rte_mempool_create(name, num_elements, get_bbdev_op_size(type),
 901                         cache_size, sizeof(struct rte_bbdev_op_pool_private),
 902                         NULL, NULL, bbdev_op_init, &type, socket_id, 0);
 903         if (mp == NULL) {
 904                 rte_bbdev_log(ERR,
 905                                 "Failed to create op pool %s (num ops=%u, op size=%u) with error: %s",
 906                                 name, num_elements, get_bbdev_op_size(type),
 907                                 rte_strerror(rte_errno));
 908                 return NULL;
 909         }
 910
 911         op_type_str = rte_bbdev_op_type_str(type);
 912         if (op_type_str == NULL)
 913                 return NULL;
 914
 915         rte_bbdev_log_debug(
 916                         "Op pool %s created for %u ops (type=%s, cache=%u, socket=%u, size=%u)",
 917                         name, num_elements, op_type_str, cache_size, socket_id,
 918                         get_bbdev_op_size(type));
 919
 920         priv = (struct rte_bbdev_op_pool_private *)rte_mempool_get_priv(mp);
 921         priv->type = type;
 922
 923         return mp;
 924 }
 925
 926 int
 927 rte_bbdev_callback_register(uint16_t dev_id, enum rte_bbdev_event_type event,
 928                 rte_bbdev_cb_fn cb_fn, void *cb_arg)
 929 {
 930         struct rte_bbdev_callback *user_cb;
 931         struct rte_bbdev *dev = get_dev(dev_id);
 932         VALID_DEV_OR_RET_ERR(dev, dev_id);
 933
 934         if (event >= RTE_BBDEV_EVENT_MAX) {
 935                 rte_bbdev_log(ERR,
 936                                 "Invalid event type (%u), should be less than %u",
 937                                 event, RTE_BBDEV_EVENT_MAX);
 938                 return -EINVAL;
 939         }
 940
 941         if (cb_fn == NULL) {
 942                 rte_bbdev_log(ERR, "NULL callback function");
 943                 return -EINVAL;
 944         }
 945
 946         rte_spinlock_lock(&rte_bbdev_cb_lock);
 947
 948         TAILQ_FOREACH(user_cb, &(dev->list_cbs), next) {
 949                 if (user_cb->cb_fn == cb_fn &&
 950                                 user_cb->cb_arg == cb_arg &&
 951                                 user_cb->event == event)
 952                         break;
 953         }
 954
 955         /* create a new callback. */
 956         if (user_cb == NULL) {
 957                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
 958                                 sizeof(struct rte_bbdev_callback), 0);
 959                 if (user_cb != NULL) {
 960                         user_cb->cb_fn = cb_fn;
 961                         user_cb->cb_arg = cb_arg;
 962                         user_cb->event = event;
 963                         TAILQ_INSERT_TAIL(&(dev->list_cbs), user_cb, next);
 964                 }
 965         }
 966
 967         rte_spinlock_unlock(&rte_bbdev_cb_lock);
 968         return (user_cb == NULL) ? -ENOMEM : 0;
 969 }
 970
 971 int
 972 rte_bbdev_callback_unregister(uint16_t dev_id, enum rte_bbdev_event_type event,
 973                 rte_bbdev_cb_fn cb_fn, void *cb_arg)
 974 {
 975         int ret = 0;
 976         struct rte_bbdev_callback *cb, *next;
 977         struct rte_bbdev *dev = get_dev(dev_id);
 978         VALID_DEV_OR_RET_ERR(dev, dev_id);
 979
 980         if (event >= RTE_BBDEV_EVENT_MAX) {
 981                 rte_bbdev_log(ERR,
 982                                 "Invalid event type (%u), should be less than %u",
 983                                 event, RTE_BBDEV_EVENT_MAX);
 984                 return -EINVAL;
 985         }
 986
 987         if (cb_fn == NULL) {
 988                 rte_bbdev_log(ERR,
 989                                 "NULL callback function cannot be unregistered");
 990                 return -EINVAL;
 991         }
 992
 993         dev = &rte_bbdev_devices[dev_id];
 994         rte_spinlock_lock(&rte_bbdev_cb_lock);
 995
 996         for (cb = TAILQ_FIRST(&dev->list_cbs); cb != NULL; cb = next) {
 997
 998                 next = TAILQ_NEXT(cb, next);
 999
1000                 if (cb->cb_fn != cb_fn || cb->event != event ||
1001                                 (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
1002                         continue;
1003
1004                 /* If this callback is not executing right now, remove it. */
1005                 if (cb->active == 0) {
1006                         TAILQ_REMOVE(&(dev->list_cbs), cb, next);
1007                         rte_free(cb);
1008                 } else
1009                         ret = -EAGAIN;
1010         }
1011
1012         rte_spinlock_unlock(&rte_bbdev_cb_lock);
1013         return ret;
1014 }
1015
1016 void
1017 rte_bbdev_pmd_callback_process(struct rte_bbdev *dev,
1018         enum rte_bbdev_event_type event, void *ret_param)
1019 {
1020         struct rte_bbdev_callback *cb_lst;
1021         struct rte_bbdev_callback dev_cb;
1022
1023         if (dev == NULL) {
1024                 rte_bbdev_log(ERR, "NULL device");
1025                 return;
1026         }
1027
1028         if (dev->data == NULL) {
1029                 rte_bbdev_log(ERR, "NULL data structure");
1030                 return;
1031         }
1032
1033         if (event >= RTE_BBDEV_EVENT_MAX) {
1034                 rte_bbdev_log(ERR,
1035                                 "Invalid event type (%u), should be less than %u",
1036                                 event, RTE_BBDEV_EVENT_MAX);
1037                 return;
1038         }
1039
1040         rte_spinlock_lock(&rte_bbdev_cb_lock);
1041         TAILQ_FOREACH(cb_lst, &(dev->list_cbs), next) {
1042                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1043                         continue;
1044                 dev_cb = *cb_lst;
1045                 cb_lst->active = 1;
1046                 if (ret_param != NULL)
1047                         dev_cb.ret_param = ret_param;
1048
1049                 rte_spinlock_unlock(&rte_bbdev_cb_lock);
1050                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1051                                 dev_cb.cb_arg, dev_cb.ret_param);
1052                 rte_spinlock_lock(&rte_bbdev_cb_lock);
1053                 cb_lst->active = 0;
1054         }
1055         rte_spinlock_unlock(&rte_bbdev_cb_lock);
1056 }
1057
1058 int
1059 rte_bbdev_queue_intr_enable(uint16_t dev_id, uint16_t queue_id)
1060 {
1061         struct rte_bbdev *dev = get_dev(dev_id);
1062         VALID_DEV_OR_RET_ERR(dev, dev_id);
1063         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1064         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1065         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_enable, dev_id);
1066         return dev->dev_ops->queue_intr_enable(dev, queue_id);
1067 }
1068
1069 int
1070 rte_bbdev_queue_intr_disable(uint16_t dev_id, uint16_t queue_id)
1071 {
1072         struct rte_bbdev *dev = get_dev(dev_id);
1073         VALID_DEV_OR_RET_ERR(dev, dev_id);
1074         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1075         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1076         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_disable, dev_id);
1077         return dev->dev_ops->queue_intr_disable(dev, queue_id);
1078 }
1079
1080 int
1081 rte_bbdev_queue_intr_ctl(uint16_t dev_id, uint16_t queue_id, int epfd, int op,
1082                 void *data)
1083 {
1084         uint32_t vec;
1085         struct rte_bbdev *dev = get_dev(dev_id);
1086         struct rte_intr_handle *intr_handle;
1087         int ret;
1088
1089         VALID_DEV_OR_RET_ERR(dev, dev_id);
1090         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1091
1092         intr_handle = dev->intr_handle;
1093         if (intr_handle == NULL) {
1094                 rte_bbdev_log(ERR, "Device %u intr handle unset\n", dev_id);
1095                 return -ENOTSUP;
1096         }
1097
1098         if (queue_id >= RTE_MAX_RXTX_INTR_VEC_ID) {
1099                 rte_bbdev_log(ERR, "Device %u queue_id %u is too big\n",
1100                                 dev_id, queue_id);
1101                 return -ENOTSUP;
1102         }
1103
1104         vec = rte_intr_vec_list_index_get(intr_handle, queue_id);
1105         ret = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
1106         if (ret && (ret != -EEXIST)) {
1107                 rte_bbdev_log(ERR,
1108                                 "dev %u q %u int ctl error op %d epfd %d vec %u\n",
1109                                 dev_id, queue_id, op, epfd, vec);
1110                 return ret;
1111         }
1112
1113         return 0;
1114 }
1115
1116
1117 const char *
1118 rte_bbdev_op_type_str(enum rte_bbdev_op_type op_type)
1119 {
1120         static const char * const op_types[] = {
1121                 "RTE_BBDEV_OP_NONE",
1122                 "RTE_BBDEV_OP_TURBO_DEC",
1123                 "RTE_BBDEV_OP_TURBO_ENC",
1124                 "RTE_BBDEV_OP_LDPC_DEC",
1125                 "RTE_BBDEV_OP_LDPC_ENC",
1126         };
1127
1128         if (op_type < RTE_BBDEV_OP_TYPE_COUNT)
1129                 return op_types[op_type];
1130
1131         rte_bbdev_log(ERR, "Invalid operation type");
1132         return NULL;
1133 }