lib/librte_cryptodev/rte_cryptodev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/types.h>
  34 #include <sys/queue.h>
  35 #include <ctype.h>
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <stdarg.h>
  40 #include <errno.h>
  41 #include <stdint.h>
  42 #include <inttypes.h>
  43 #include <netinet/in.h>
  44
  45 #include <rte_byteorder.h>
  46 #include <rte_log.h>
  47 #include <rte_debug.h>
  48 #include <rte_dev.h>
  49 #include <rte_interrupts.h>
  50 #include <rte_pci.h>
  51 #include <rte_memory.h>
  52 #include <rte_memcpy.h>
  53 #include <rte_memzone.h>
  54 #include <rte_launch.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_per_lcore.h>
  58 #include <rte_lcore.h>
  59 #include <rte_atomic.h>
  60 #include <rte_branch_prediction.h>
  61 #include <rte_common.h>
  62 #include <rte_ring.h>
  63 #include <rte_mempool.h>
  64 #include <rte_malloc.h>
  65 #include <rte_mbuf.h>
  66 #include <rte_errno.h>
  67 #include <rte_spinlock.h>
  68 #include <rte_string_fns.h>
  69
  70 #include "rte_crypto.h"
  71 #include "rte_cryptodev.h"
  72 #include "rte_cryptodev_pmd.h"
  73
  74 struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
  75
  76 struct rte_cryptodev *rte_cryptodevs = &rte_crypto_devices[0];
  77
  78 static struct rte_cryptodev_global cryptodev_globals = {
  79                 .devs                   = &rte_crypto_devices[0],
  80                 .data                   = { NULL },
  81                 .nb_devs                = 0,
  82                 .max_devs               = RTE_CRYPTO_MAX_DEVS
  83 };
  84
  85 struct rte_cryptodev_global *rte_cryptodev_globals = &cryptodev_globals;
  86
  87 /* spinlock for crypto device callbacks */
  88 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  89
  90
  91 /**
  92  * The user application callback description.
  93  *
  94  * It contains callback address to be registered by user application,
  95  * the pointer to the parameters for callback, and the event type.
  96  */
  97 struct rte_cryptodev_callback {
  98         TAILQ_ENTRY(rte_cryptodev_callback) next; /**< Callbacks list */
  99         rte_cryptodev_cb_fn cb_fn;              /**< Callback address */
 100         void *cb_arg;                           /**< Parameter for callback */
 101         enum rte_cryptodev_event_type event;    /**< Interrupt event type */
 102         uint32_t active;                        /**< Callback is executing */
 103 };
 104
 105 int
 106 rte_cryptodev_create_vdev(const char *name, const char *args)
 107 {
 108         return rte_eal_vdev_init(name, args);
 109 }
 110
 111 int
 112 rte_cryptodev_get_dev_id(const char *name) {
 113         unsigned i;
 114
 115         if (name == NULL)
 116                 return -1;
 117
 118         for (i = 0; i < rte_cryptodev_globals->max_devs; i++)
 119                 if ((strcmp(rte_cryptodev_globals->devs[i].data->name, name)
 120                                 == 0) &&
 121                                 (rte_cryptodev_globals->devs[i].attached ==
 122                                                 RTE_CRYPTODEV_ATTACHED))
 123                         return i;
 124
 125         return -1;
 126 }
 127
 128 uint8_t
 129 rte_cryptodev_count(void)
 130 {
 131         return rte_cryptodev_globals->nb_devs;
 132 }
 133
 134 uint8_t
 135 rte_cryptodev_count_devtype(enum rte_cryptodev_type type)
 136 {
 137         uint8_t i, dev_count = 0;
 138
 139         for (i = 0; i < rte_cryptodev_globals->max_devs; i++)
 140                 if (rte_cryptodev_globals->devs[i].dev_type == type &&
 141                         rte_cryptodev_globals->devs[i].attached ==
 142                                         RTE_CRYPTODEV_ATTACHED)
 143                         dev_count++;
 144
 145         return dev_count;
 146 }
 147
 148 int
 149 rte_cryptodev_socket_id(uint8_t dev_id)
 150 {
 151         struct rte_cryptodev *dev;
 152
 153         if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
 154                 return -1;
 155
 156         dev = rte_cryptodev_pmd_get_dev(dev_id);
 157
 158         return dev->data->socket_id;
 159 }
 160
 161 static inline int
 162 rte_cryptodev_data_alloc(uint8_t dev_id, struct rte_cryptodev_data **data,
 163                 int socket_id)
 164 {
 165         char mz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 166         const struct rte_memzone *mz;
 167         int n;
 168
 169         /* generate memzone name */
 170         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 171         if (n >= (int)sizeof(mz_name))
 172                 return -EINVAL;
 173
 174         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 175                 mz = rte_memzone_reserve(mz_name,
 176                                 sizeof(struct rte_cryptodev_data),
 177                                 socket_id, 0);
 178         } else
 179                 mz = rte_memzone_lookup(mz_name);
 180
 181         if (mz == NULL)
 182                 return -ENOMEM;
 183
 184         *data = mz->addr;
 185         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 186                 memset(*data, 0, sizeof(struct rte_cryptodev_data));
 187
 188         return 0;
 189 }
 190
 191 static uint8_t
 192 rte_cryptodev_find_free_device_index(void)
 193 {
 194         uint8_t dev_id;
 195
 196         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++) {
 197                 if (rte_crypto_devices[dev_id].attached ==
 198                                 RTE_CRYPTODEV_DETACHED)
 199                         return dev_id;
 200         }
 201         return RTE_CRYPTO_MAX_DEVS;
 202 }
 203
 204 struct rte_cryptodev *
 205 rte_cryptodev_pmd_allocate(const char *name, enum pmd_type type, int socket_id)
 206 {
 207         struct rte_cryptodev *cryptodev;
 208         uint8_t dev_id;
 209
 210         if (rte_cryptodev_pmd_get_named_dev(name) != NULL) {
 211                 CDEV_LOG_ERR("Crypto device with name %s already "
 212                                 "allocated!", name);
 213                 return NULL;
 214         }
 215
 216         dev_id = rte_cryptodev_find_free_device_index();
 217         if (dev_id == RTE_CRYPTO_MAX_DEVS) {
 218                 CDEV_LOG_ERR("Reached maximum number of crypto devices");
 219                 return NULL;
 220         }
 221
 222         cryptodev = rte_cryptodev_pmd_get_dev(dev_id);
 223
 224         if (cryptodev->data == NULL) {
 225                 struct rte_cryptodev_data *cryptodev_data =
 226                                 cryptodev_globals.data[dev_id];
 227
 228                 int retval = rte_cryptodev_data_alloc(dev_id, &cryptodev_data,
 229                                 socket_id);
 230
 231                 if (retval < 0 || cryptodev_data == NULL)
 232                         return NULL;
 233
 234                 cryptodev->data = cryptodev_data;
 235
 236                 snprintf(cryptodev->data->name, RTE_CRYPTODEV_NAME_MAX_LEN,
 237                                 "%s", name);
 238
 239                 cryptodev->data->dev_id = dev_id;
 240                 cryptodev->data->socket_id = socket_id;
 241                 cryptodev->data->dev_started = 0;
 242
 243                 cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 244                 cryptodev->pmd_type = type;
 245
 246                 cryptodev_globals.nb_devs++;
 247         }
 248
 249         return cryptodev;
 250 }
 251
 252 static inline int
 253 rte_cryptodev_create_unique_device_name(char *name, size_t size,
 254                 struct rte_pci_device *pci_dev)
 255 {
 256         int ret;
 257
 258         if ((name == NULL) || (pci_dev == NULL))
 259                 return -EINVAL;
 260
 261         ret = snprintf(name, size, "%d:%d.%d",
 262                         pci_dev->addr.bus, pci_dev->addr.devid,
 263                         pci_dev->addr.function);
 264         if (ret < 0)
 265                 return ret;
 266         return 0;
 267 }
 268
 269 int
 270 rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 271 {
 272         int ret;
 273
 274         if (cryptodev == NULL)
 275                 return -EINVAL;
 276
 277         ret = rte_cryptodev_close(cryptodev->data->dev_id);
 278         if (ret < 0)
 279                 return ret;
 280
 281         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 282         cryptodev_globals.nb_devs--;
 283         return 0;
 284 }
 285
 286 struct rte_cryptodev *
 287 rte_cryptodev_pmd_virtual_dev_init(const char *name, size_t dev_private_size,
 288                 int socket_id)
 289 {
 290         struct rte_cryptodev *cryptodev;
 291
 292         /* allocate device structure */
 293         cryptodev = rte_cryptodev_pmd_allocate(name, PMD_VDEV, socket_id);
 294         if (cryptodev == NULL)
 295                 return NULL;
 296
 297         /* allocate private device structure */
 298         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 299                 cryptodev->data->dev_private =
 300                                 rte_zmalloc_socket("cryptodev device private",
 301                                                 dev_private_size,
 302                                                 RTE_CACHE_LINE_SIZE,
 303                                                 socket_id);
 304
 305                 if (cryptodev->data->dev_private == NULL)
 306                         rte_panic("Cannot allocate memzone for private device"
 307                                         " data");
 308         }
 309
 310         /* initialise user call-back tail queue */
 311         TAILQ_INIT(&(cryptodev->link_intr_cbs));
 312
 313         return cryptodev;
 314 }
 315
 316 static int
 317 rte_cryptodev_init(struct rte_pci_driver *pci_drv,
 318                 struct rte_pci_device *pci_dev)
 319 {
 320         struct rte_cryptodev_driver *cryptodrv;
 321         struct rte_cryptodev *cryptodev;
 322
 323         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 324
 325         int retval;
 326
 327         cryptodrv = (struct rte_cryptodev_driver *)pci_drv;
 328         if (cryptodrv == NULL)
 329                 return -ENODEV;
 330
 331         /* Create unique Crypto device name using PCI address */
 332         rte_cryptodev_create_unique_device_name(cryptodev_name,
 333                         sizeof(cryptodev_name), pci_dev);
 334
 335         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, PMD_PDEV,
 336                         rte_socket_id());
 337         if (cryptodev == NULL)
 338                 return -ENOMEM;
 339
 340         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 341                 cryptodev->data->dev_private =
 342                                 rte_zmalloc_socket(
 343                                                 "cryptodev private structure",
 344                                                 cryptodrv->dev_private_size,
 345                                                 RTE_CACHE_LINE_SIZE,
 346                                                 rte_socket_id());
 347
 348                 if (cryptodev->data->dev_private == NULL)
 349                         rte_panic("Cannot allocate memzone for private "
 350                                         "device data");
 351         }
 352
 353         cryptodev->pci_dev = pci_dev;
 354         cryptodev->driver = cryptodrv;
 355
 356         /* init user callbacks */
 357         TAILQ_INIT(&(cryptodev->link_intr_cbs));
 358
 359         /* Invoke PMD device initialization function */
 360         retval = (*cryptodrv->cryptodev_init)(cryptodrv, cryptodev);
 361         if (retval == 0)
 362                 return 0;
 363
 364         CDEV_LOG_ERR("driver %s: crypto_dev_init(vendor_id=0x%x device_id=0x%x)"
 365                         " failed", pci_drv->name,
 366                         (unsigned) pci_dev->id.vendor_id,
 367                         (unsigned) pci_dev->id.device_id);
 368
 369         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 370                 rte_free(cryptodev->data->dev_private);
 371
 372         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 373         cryptodev_globals.nb_devs--;
 374
 375         return -ENXIO;
 376 }
 377
 378 static int
 379 rte_cryptodev_uninit(struct rte_pci_device *pci_dev)
 380 {
 381         const struct rte_cryptodev_driver *cryptodrv;
 382         struct rte_cryptodev *cryptodev;
 383         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 384         int ret;
 385
 386         if (pci_dev == NULL)
 387                 return -EINVAL;
 388
 389         /* Create unique device name using PCI address */
 390         rte_cryptodev_create_unique_device_name(cryptodev_name,
 391                         sizeof(cryptodev_name), pci_dev);
 392
 393         cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
 394         if (cryptodev == NULL)
 395                 return -ENODEV;
 396
 397         cryptodrv = (const struct rte_cryptodev_driver *)pci_dev->driver;
 398         if (cryptodrv == NULL)
 399                 return -ENODEV;
 400
 401         /* Invoke PMD device uninit function */
 402         if (*cryptodrv->cryptodev_uninit) {
 403                 ret = (*cryptodrv->cryptodev_uninit)(cryptodrv, cryptodev);
 404                 if (ret)
 405                         return ret;
 406         }
 407
 408         /* free crypto device */
 409         rte_cryptodev_pmd_release_device(cryptodev);
 410
 411         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 412                 rte_free(cryptodev->data->dev_private);
 413
 414         cryptodev->pci_dev = NULL;
 415         cryptodev->driver = NULL;
 416         cryptodev->data = NULL;
 417
 418         return 0;
 419 }
 420
 421 int
 422 rte_cryptodev_pmd_driver_register(struct rte_cryptodev_driver *cryptodrv,
 423                 enum pmd_type type)
 424 {
 425         /* Call crypto device initialization directly if device is virtual */
 426         if (type == PMD_VDEV)
 427                 return rte_cryptodev_init((struct rte_pci_driver *)cryptodrv,
 428                                 NULL);
 429
 430         /*
 431          * Register PCI driver for physical device intialisation during
 432          * PCI probing
 433          */
 434         cryptodrv->pci_drv.devinit = rte_cryptodev_init;
 435         cryptodrv->pci_drv.devuninit = rte_cryptodev_uninit;
 436
 437         rte_eal_pci_register(&cryptodrv->pci_drv);
 438
 439         return 0;
 440 }
 441
 442
 443 uint16_t
 444 rte_cryptodev_queue_pair_count(uint8_t dev_id)
 445 {
 446         struct rte_cryptodev *dev;
 447
 448         dev = &rte_crypto_devices[dev_id];
 449         return dev->data->nb_queue_pairs;
 450 }
 451
 452 static int
 453 rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 454                 int socket_id)
 455 {
 456         struct rte_cryptodev_info dev_info;
 457         void **qp;
 458         unsigned i;
 459
 460         if ((dev == NULL) || (nb_qpairs < 1)) {
 461                 CDEV_LOG_ERR("invalid param: dev %p, nb_queues %u",
 462                                                         dev, nb_qpairs);
 463                 return -EINVAL;
 464         }
 465
 466         CDEV_LOG_DEBUG("Setup %d queues pairs on device %u",
 467                         nb_qpairs, dev->data->dev_id);
 468
 469         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 470
 471         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 472         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 473
 474         if (nb_qpairs > (dev_info.max_nb_queue_pairs)) {
 475                 CDEV_LOG_ERR("Invalid num queue_pairs (%u) for dev %u",
 476                                 nb_qpairs, dev->data->dev_id);
 477             return -EINVAL;
 478         }
 479
 480         if (dev->data->queue_pairs == NULL) { /* first time configuration */
 481                 dev->data->queue_pairs = rte_zmalloc_socket(
 482                                 "cryptodev->queue_pairs",
 483                                 sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
 484                                 RTE_CACHE_LINE_SIZE, socket_id);
 485
 486                 if (dev->data->queue_pairs == NULL) {
 487                         dev->data->nb_queue_pairs = 0;
 488                         CDEV_LOG_ERR("failed to get memory for qp meta data, "
 489                                                         "nb_queues %u",
 490                                                         nb_qpairs);
 491                         return -(ENOMEM);
 492                 }
 493         } else { /* re-configure */
 494                 int ret;
 495                 uint16_t old_nb_queues = dev->data->nb_queue_pairs;
 496
 497                 qp = dev->data->queue_pairs;
 498
 499                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
 500                                 -ENOTSUP);
 501
 502                 for (i = nb_qpairs; i < old_nb_queues; i++) {
 503                         ret = (*dev->dev_ops->queue_pair_release)(dev, i);
 504                         if (ret < 0)
 505                                 return ret;
 506                 }
 507
 508                 qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
 509                                 RTE_CACHE_LINE_SIZE);
 510                 if (qp == NULL) {
 511                         CDEV_LOG_ERR("failed to realloc qp meta data,"
 512                                                 " nb_queues %u", nb_qpairs);
 513                         return -(ENOMEM);
 514                 }
 515
 516                 if (nb_qpairs > old_nb_queues) {
 517                         uint16_t new_qs = nb_qpairs - old_nb_queues;
 518
 519                         memset(qp + old_nb_queues, 0,
 520                                 sizeof(qp[0]) * new_qs);
 521                 }
 522
 523                 dev->data->queue_pairs = qp;
 524
 525         }
 526         dev->data->nb_queue_pairs = nb_qpairs;
 527         return 0;
 528 }
 529
 530 int
 531 rte_cryptodev_queue_pair_start(uint8_t dev_id, uint16_t queue_pair_id)
 532 {
 533         struct rte_cryptodev *dev;
 534
 535         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 536                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 537                 return -EINVAL;
 538         }
 539
 540         dev = &rte_crypto_devices[dev_id];
 541         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 542                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 543                 return -EINVAL;
 544         }
 545
 546         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_start, -ENOTSUP);
 547
 548         return dev->dev_ops->queue_pair_start(dev, queue_pair_id);
 549
 550 }
 551
 552 int
 553 rte_cryptodev_queue_pair_stop(uint8_t dev_id, uint16_t queue_pair_id)
 554 {
 555         struct rte_cryptodev *dev;
 556
 557         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 558                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 559                 return -EINVAL;
 560         }
 561
 562         dev = &rte_crypto_devices[dev_id];
 563         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 564                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 565                 return -EINVAL;
 566         }
 567
 568         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_stop, -ENOTSUP);
 569
 570         return dev->dev_ops->queue_pair_stop(dev, queue_pair_id);
 571
 572 }
 573
 574 static int
 575 rte_cryptodev_sym_session_pool_create(struct rte_cryptodev *dev,
 576                 unsigned nb_objs, unsigned obj_cache_size, int socket_id);
 577
 578 int
 579 rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
 580 {
 581         struct rte_cryptodev *dev;
 582         int diag;
 583
 584         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 585                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 586                 return -EINVAL;
 587         }
 588
 589         dev = &rte_crypto_devices[dev_id];
 590
 591         if (dev->data->dev_started) {
 592                 CDEV_LOG_ERR(
 593                     "device %d must be stopped to allow configuration", dev_id);
 594                 return -EBUSY;
 595         }
 596
 597         /* Setup new number of queue pairs and reconfigure device. */
 598         diag = rte_cryptodev_queue_pairs_config(dev, config->nb_queue_pairs,
 599                         config->socket_id);
 600         if (diag != 0) {
 601                 CDEV_LOG_ERR("dev%d rte_crypto_dev_queue_pairs_config = %d",
 602                                 dev_id, diag);
 603                 return diag;
 604         }
 605
 606         /* Setup Session mempool for device */
 607         return rte_cryptodev_sym_session_pool_create(dev,
 608                         config->session_mp.nb_objs,
 609                         config->session_mp.cache_size,
 610                         config->socket_id);
 611 }
 612
 613
 614 int
 615 rte_cryptodev_start(uint8_t dev_id)
 616 {
 617         struct rte_cryptodev *dev;
 618         int diag;
 619
 620         CDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id);
 621
 622         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 623                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 624                 return -EINVAL;
 625         }
 626
 627         dev = &rte_crypto_devices[dev_id];
 628
 629         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
 630
 631         if (dev->data->dev_started != 0) {
 632                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already started",
 633                         dev_id);
 634                 return 0;
 635         }
 636
 637         diag = (*dev->dev_ops->dev_start)(dev);
 638         if (diag == 0)
 639                 dev->data->dev_started = 1;
 640         else
 641                 return diag;
 642
 643         return 0;
 644 }
 645
 646 void
 647 rte_cryptodev_stop(uint8_t dev_id)
 648 {
 649         struct rte_cryptodev *dev;
 650
 651         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 652                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 653                 return;
 654         }
 655
 656         dev = &rte_crypto_devices[dev_id];
 657
 658         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
 659
 660         if (dev->data->dev_started == 0) {
 661                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already stopped",
 662                         dev_id);
 663                 return;
 664         }
 665
 666         dev->data->dev_started = 0;
 667         (*dev->dev_ops->dev_stop)(dev);
 668 }
 669
 670 int
 671 rte_cryptodev_close(uint8_t dev_id)
 672 {
 673         struct rte_cryptodev *dev;
 674         int retval;
 675
 676         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 677                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 678                 return -1;
 679         }
 680
 681         dev = &rte_crypto_devices[dev_id];
 682
 683         /* Device must be stopped before it can be closed */
 684         if (dev->data->dev_started == 1) {
 685                 CDEV_LOG_ERR("Device %u must be stopped before closing",
 686                                 dev_id);
 687                 return -EBUSY;
 688         }
 689
 690         /* We can't close the device if there are outstanding sessions in use */
 691         if (dev->data->session_pool != NULL) {
 692                 if (!rte_mempool_full(dev->data->session_pool)) {
 693                         CDEV_LOG_ERR("dev_id=%u close failed, session mempool "
 694                                         "has sessions still in use, free "
 695                                         "all sessions before calling close",
 696                                         (unsigned)dev_id);
 697                         return -EBUSY;
 698                 }
 699         }
 700
 701         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
 702         retval = (*dev->dev_ops->dev_close)(dev);
 703
 704         if (retval < 0)
 705                 return retval;
 706
 707         return 0;
 708 }
 709
 710 int
 711 rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
 712                 const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
 713 {
 714         struct rte_cryptodev *dev;
 715
 716         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 717                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 718                 return -EINVAL;
 719         }
 720
 721         dev = &rte_crypto_devices[dev_id];
 722         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 723                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 724                 return -EINVAL;
 725         }
 726
 727         if (dev->data->dev_started) {
 728                 CDEV_LOG_ERR(
 729                     "device %d must be stopped to allow configuration", dev_id);
 730                 return -EBUSY;
 731         }
 732
 733         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
 734
 735         return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id, qp_conf,
 736                         socket_id);
 737 }
 738
 739
 740 int
 741 rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats)
 742 {
 743         struct rte_cryptodev *dev;
 744
 745         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 746                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
 747                 return -ENODEV;
 748         }
 749
 750         if (stats == NULL) {
 751                 CDEV_LOG_ERR("Invalid stats ptr");
 752                 return -EINVAL;
 753         }
 754
 755         dev = &rte_crypto_devices[dev_id];
 756         memset(stats, 0, sizeof(*stats));
 757
 758         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
 759         (*dev->dev_ops->stats_get)(dev, stats);
 760         return 0;
 761 }
 762
 763 void
 764 rte_cryptodev_stats_reset(uint8_t dev_id)
 765 {
 766         struct rte_cryptodev *dev;
 767
 768         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 769                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 770                 return;
 771         }
 772
 773         dev = &rte_crypto_devices[dev_id];
 774
 775         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
 776         (*dev->dev_ops->stats_reset)(dev);
 777 }
 778
 779
 780 void
 781 rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
 782 {
 783         struct rte_cryptodev *dev;
 784
 785         if (dev_id >= cryptodev_globals.nb_devs) {
 786                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
 787                 return;
 788         }
 789
 790         dev = &rte_crypto_devices[dev_id];
 791
 792         memset(dev_info, 0, sizeof(struct rte_cryptodev_info));
 793
 794         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
 795         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
 796
 797         dev_info->pci_dev = dev->pci_dev;
 798         if (dev->driver)
 799                 dev_info->driver_name = dev->driver->pci_drv.name;
 800 }
 801
 802
 803 int
 804 rte_cryptodev_callback_register(uint8_t dev_id,
 805                         enum rte_cryptodev_event_type event,
 806                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
 807 {
 808         struct rte_cryptodev *dev;
 809         struct rte_cryptodev_callback *user_cb;
 810
 811         if (!cb_fn)
 812                 return -EINVAL;
 813
 814         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 815                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 816                 return -EINVAL;
 817         }
 818
 819         dev = &rte_crypto_devices[dev_id];
 820         rte_spinlock_lock(&rte_cryptodev_cb_lock);
 821
 822         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
 823                 if (user_cb->cb_fn == cb_fn &&
 824                         user_cb->cb_arg == cb_arg &&
 825                         user_cb->event == event) {
 826                         break;
 827                 }
 828         }
 829
 830         /* create a new callback. */
 831         if (user_cb == NULL) {
 832                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
 833                                 sizeof(struct rte_cryptodev_callback), 0);
 834                 if (user_cb != NULL) {
 835                         user_cb->cb_fn = cb_fn;
 836                         user_cb->cb_arg = cb_arg;
 837                         user_cb->event = event;
 838                         TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
 839                 }
 840         }
 841
 842         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 843         return (user_cb == NULL) ? -ENOMEM : 0;
 844 }
 845
 846 int
 847 rte_cryptodev_callback_unregister(uint8_t dev_id,
 848                         enum rte_cryptodev_event_type event,
 849                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
 850 {
 851         int ret;
 852         struct rte_cryptodev *dev;
 853         struct rte_cryptodev_callback *cb, *next;
 854
 855         if (!cb_fn)
 856                 return -EINVAL;
 857
 858         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 859                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 860                 return -EINVAL;
 861         }
 862
 863         dev = &rte_crypto_devices[dev_id];
 864         rte_spinlock_lock(&rte_cryptodev_cb_lock);
 865
 866         ret = 0;
 867         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
 868
 869                 next = TAILQ_NEXT(cb, next);
 870
 871                 if (cb->cb_fn != cb_fn || cb->event != event ||
 872                                 (cb->cb_arg != (void *)-1 &&
 873                                 cb->cb_arg != cb_arg))
 874                         continue;
 875
 876                 /*
 877                  * if this callback is not executing right now,
 878                  * then remove it.
 879                  */
 880                 if (cb->active == 0) {
 881                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
 882                         rte_free(cb);
 883                 } else {
 884                         ret = -EAGAIN;
 885                 }
 886         }
 887
 888         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 889         return ret;
 890 }
 891
 892 void
 893 rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev,
 894         enum rte_cryptodev_event_type event)
 895 {
 896         struct rte_cryptodev_callback *cb_lst;
 897         struct rte_cryptodev_callback dev_cb;
 898
 899         rte_spinlock_lock(&rte_cryptodev_cb_lock);
 900         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
 901                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
 902                         continue;
 903                 dev_cb = *cb_lst;
 904                 cb_lst->active = 1;
 905                 rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 906                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
 907                                                 dev_cb.cb_arg);
 908                 rte_spinlock_lock(&rte_cryptodev_cb_lock);
 909                 cb_lst->active = 0;
 910         }
 911         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 912 }
 913
 914
 915 static void
 916 rte_cryptodev_sym_session_init(struct rte_mempool *mp,
 917                 void *opaque_arg,
 918                 void *_sess,
 919                 __rte_unused unsigned i)
 920 {
 921         struct rte_cryptodev_sym_session *sess = _sess;
 922         struct rte_cryptodev *dev = opaque_arg;
 923
 924         memset(sess, 0, mp->elt_size);
 925
 926         sess->dev_id = dev->data->dev_id;
 927         sess->type = dev->dev_type;
 928         sess->mp = mp;
 929
 930         if (dev->dev_ops->session_initialize)
 931                 (*dev->dev_ops->session_initialize)(mp, sess->_private);
 932 }
 933
 934 static int
 935 rte_cryptodev_sym_session_pool_create(struct rte_cryptodev *dev,
 936                 unsigned nb_objs, unsigned obj_cache_size, int socket_id)
 937 {
 938         char mp_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 939         unsigned priv_sess_size;
 940
 941         unsigned n = snprintf(mp_name, sizeof(mp_name), "cdev_%d_sess_mp",
 942                         dev->data->dev_id);
 943         if (n > sizeof(mp_name)) {
 944                 CDEV_LOG_ERR("Unable to create unique name for session mempool");
 945                 return -ENOMEM;
 946         }
 947
 948         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_get_size, -ENOTSUP);
 949         priv_sess_size = (*dev->dev_ops->session_get_size)(dev);
 950         if (priv_sess_size == 0) {
 951                 CDEV_LOG_ERR("%s returned and invalid private session size ",
 952                                                 dev->data->name);
 953                 return -ENOMEM;
 954         }
 955
 956         unsigned elt_size = sizeof(struct rte_cryptodev_sym_session) +
 957                         priv_sess_size;
 958
 959         dev->data->session_pool = rte_mempool_lookup(mp_name);
 960         if (dev->data->session_pool != NULL) {
 961                 if ((dev->data->session_pool->elt_size != elt_size) ||
 962                                 (dev->data->session_pool->cache_size <
 963                                 obj_cache_size) ||
 964                                 (dev->data->session_pool->size < nb_objs)) {
 965
 966                         CDEV_LOG_ERR("%s mempool already exists with different"
 967                                         " initialization parameters", mp_name);
 968                         dev->data->session_pool = NULL;
 969                         return -ENOMEM;
 970                 }
 971         } else {
 972                 dev->data->session_pool = rte_mempool_create(
 973                                 mp_name, /* mempool name */
 974                                 nb_objs, /* number of elements*/
 975                                 elt_size, /* element size*/
 976                                 obj_cache_size, /* Cache size*/
 977                                 0, /* private data size */
 978                                 NULL, /* obj initialization constructor */
 979                                 NULL, /* obj initialization constructor arg */
 980                                 rte_cryptodev_sym_session_init,
 981                                 /**< obj constructor*/
 982                                 dev, /* obj constructor arg */
 983                                 socket_id, /* socket id */
 984                                 0); /* flags */
 985
 986                 if (dev->data->session_pool == NULL) {
 987                         CDEV_LOG_ERR("%s mempool allocation failed", mp_name);
 988                         return -ENOMEM;
 989                 }
 990         }
 991
 992         CDEV_LOG_DEBUG("%s mempool created!", mp_name);
 993         return 0;
 994 }
 995
 996 struct rte_cryptodev_sym_session *
 997 rte_cryptodev_sym_session_create(uint8_t dev_id,
 998                 struct rte_crypto_sym_xform *xform)
 999 {
1000         struct rte_cryptodev *dev;
1001         struct rte_cryptodev_sym_session *sess;
1002         void *_sess;
1003
1004         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1005                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1006                 return NULL;
1007         }
1008
1009         dev = &rte_crypto_devices[dev_id];
1010
1011         /* Allocate a session structure from the session pool */
1012         if (rte_mempool_get(dev->data->session_pool, &_sess)) {
1013                 CDEV_LOG_ERR("Couldn't get object from session mempool");
1014                 return NULL;
1015         }
1016
1017         sess = (struct rte_cryptodev_sym_session *)_sess;
1018
1019         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_configure, NULL);
1020         if (dev->dev_ops->session_configure(dev, xform, sess->_private) ==
1021                         NULL) {
1022                 CDEV_LOG_ERR("dev_id %d failed to configure session details",
1023                                 dev_id);
1024
1025                 /* Return session to mempool */
1026                 rte_mempool_put(sess->mp, _sess);
1027                 return NULL;
1028         }
1029
1030         return sess;
1031 }
1032
1033 struct rte_cryptodev_sym_session *
1034 rte_cryptodev_sym_session_free(uint8_t dev_id,
1035                 struct rte_cryptodev_sym_session *sess)
1036 {
1037         struct rte_cryptodev *dev;
1038
1039         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1040                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1041                 return sess;
1042         }
1043
1044         dev = &rte_crypto_devices[dev_id];
1045
1046         /* Check the session belongs to this device type */
1047         if (sess->type != dev->dev_type)
1048                 return sess;
1049
1050         /* Let device implementation clear session material */
1051         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_clear, sess);
1052         dev->dev_ops->session_clear(dev, (void *)sess->_private);
1053
1054         /* Return session to mempool */
1055         rte_mempool_put(sess->mp, (void *)sess);
1056
1057         return NULL;
1058 }