lib/librte_cryptodev/rte_cryptodev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/types.h>
  34 #include <sys/queue.h>
  35 #include <ctype.h>
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <stdarg.h>
  40 #include <errno.h>
  41 #include <stdint.h>
  42 #include <inttypes.h>
  43 #include <netinet/in.h>
  44
  45 #include <rte_byteorder.h>
  46 #include <rte_log.h>
  47 #include <rte_debug.h>
  48 #include <rte_dev.h>
  49 #include <rte_interrupts.h>
  50 #include <rte_pci.h>
  51 #include <rte_memory.h>
  52 #include <rte_memcpy.h>
  53 #include <rte_memzone.h>
  54 #include <rte_launch.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_per_lcore.h>
  58 #include <rte_lcore.h>
  59 #include <rte_atomic.h>
  60 #include <rte_branch_prediction.h>
  61 #include <rte_common.h>
  62 #include <rte_ring.h>
  63 #include <rte_mempool.h>
  64 #include <rte_malloc.h>
  65 #include <rte_mbuf.h>
  66 #include <rte_errno.h>
  67 #include <rte_spinlock.h>
  68 #include <rte_string_fns.h>
  69
  70 #include "rte_crypto.h"
  71 #include "rte_cryptodev.h"
  72 #include "rte_cryptodev_pmd.h"
  73
  74 struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
  75
  76 struct rte_cryptodev *rte_cryptodevs = &rte_crypto_devices[0];
  77
  78 static struct rte_cryptodev_global cryptodev_globals = {
  79                 .devs                   = &rte_crypto_devices[0],
  80                 .data                   = { NULL },
  81                 .nb_devs                = 0,
  82                 .max_devs               = RTE_CRYPTO_MAX_DEVS
  83 };
  84
  85 struct rte_cryptodev_global *rte_cryptodev_globals = &cryptodev_globals;
  86
  87 /* spinlock for crypto device callbacks */
  88 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  89
  90
  91 /**
  92  * The user application callback description.
  93  *
  94  * It contains callback address to be registered by user application,
  95  * the pointer to the parameters for callback, and the event type.
  96  */
  97 struct rte_cryptodev_callback {
  98         TAILQ_ENTRY(rte_cryptodev_callback) next; /**< Callbacks list */
  99         rte_cryptodev_cb_fn cb_fn;              /**< Callback address */
 100         void *cb_arg;                           /**< Parameter for callback */
 101         enum rte_cryptodev_event_type event;    /**< Interrupt event type */
 102         uint32_t active;                        /**< Callback is executing */
 103 };
 104
 105 int
 106 rte_cryptodev_create_vdev(const char *name, const char *args)
 107 {
 108         return rte_eal_vdev_init(name, args);
 109 }
 110
 111 int
 112 rte_cryptodev_get_dev_id(const char *name) {
 113         unsigned i;
 114
 115         if (name == NULL)
 116                 return -1;
 117
 118         for (i = 0; i < rte_cryptodev_globals->max_devs; i++)
 119                 if ((strcmp(rte_cryptodev_globals->devs[i].data->name, name)
 120                                 == 0) &&
 121                                 (rte_cryptodev_globals->devs[i].attached ==
 122                                                 RTE_CRYPTODEV_ATTACHED))
 123                         return i;
 124
 125         return -1;
 126 }
 127
 128 uint8_t
 129 rte_cryptodev_count(void)
 130 {
 131         return rte_cryptodev_globals->nb_devs;
 132 }
 133
 134 uint8_t
 135 rte_cryptodev_count_devtype(enum rte_cryptodev_type type)
 136 {
 137         uint8_t i, dev_count = 0;
 138
 139         for (i = 0; i < rte_cryptodev_globals->max_devs; i++)
 140                 if (rte_cryptodev_globals->devs[i].dev_type == type &&
 141                         rte_cryptodev_globals->devs[i].attached ==
 142                                         RTE_CRYPTODEV_ATTACHED)
 143                         dev_count++;
 144
 145         return dev_count;
 146 }
 147
 148 int
 149 rte_cryptodev_socket_id(uint8_t dev_id)
 150 {
 151         struct rte_cryptodev *dev;
 152
 153         if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
 154                 return -1;
 155
 156         dev = rte_cryptodev_pmd_get_dev(dev_id);
 157
 158         return dev->data->socket_id;
 159 }
 160
 161 static inline int
 162 rte_cryptodev_data_alloc(uint8_t dev_id, struct rte_cryptodev_data **data,
 163                 int socket_id)
 164 {
 165         char mz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 166         const struct rte_memzone *mz;
 167         int n;
 168
 169         /* generate memzone name */
 170         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 171         if (n >= (int)sizeof(mz_name))
 172                 return -EINVAL;
 173
 174         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 175                 mz = rte_memzone_reserve(mz_name,
 176                                 sizeof(struct rte_cryptodev_data),
 177                                 socket_id, 0);
 178         } else
 179                 mz = rte_memzone_lookup(mz_name);
 180
 181         if (mz == NULL)
 182                 return -ENOMEM;
 183
 184         *data = mz->addr;
 185         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 186                 memset(*data, 0, sizeof(struct rte_cryptodev_data));
 187
 188         return 0;
 189 }
 190
 191 static uint8_t
 192 rte_cryptodev_find_free_device_index(void)
 193 {
 194         uint8_t dev_id;
 195
 196         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++) {
 197                 if (rte_crypto_devices[dev_id].attached ==
 198                                 RTE_CRYPTODEV_DETACHED)
 199                         return dev_id;
 200         }
 201         return RTE_CRYPTO_MAX_DEVS;
 202 }
 203
 204 struct rte_cryptodev *
 205 rte_cryptodev_pmd_allocate(const char *name, enum pmd_type type, int socket_id)
 206 {
 207         struct rte_cryptodev *cryptodev;
 208         uint8_t dev_id;
 209
 210         if (rte_cryptodev_pmd_get_named_dev(name) != NULL) {
 211                 CDEV_LOG_ERR("Crypto device with name %s already "
 212                                 "allocated!", name);
 213                 return NULL;
 214         }
 215
 216         dev_id = rte_cryptodev_find_free_device_index();
 217         if (dev_id == RTE_CRYPTO_MAX_DEVS) {
 218                 CDEV_LOG_ERR("Reached maximum number of crypto devices");
 219                 return NULL;
 220         }
 221
 222         cryptodev = rte_cryptodev_pmd_get_dev(dev_id);
 223
 224         if (cryptodev->data == NULL) {
 225                 struct rte_cryptodev_data *cryptodev_data =
 226                                 cryptodev_globals.data[dev_id];
 227
 228                 int retval = rte_cryptodev_data_alloc(dev_id, &cryptodev_data,
 229                                 socket_id);
 230
 231                 if (retval < 0 || cryptodev_data == NULL)
 232                         return NULL;
 233
 234                 cryptodev->data = cryptodev_data;
 235
 236                 snprintf(cryptodev->data->name, RTE_CRYPTODEV_NAME_MAX_LEN,
 237                                 "%s", name);
 238
 239                 cryptodev->data->dev_id = dev_id;
 240                 cryptodev->data->socket_id = socket_id;
 241                 cryptodev->data->dev_started = 0;
 242
 243                 cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 244                 cryptodev->pmd_type = type;
 245
 246                 cryptodev_globals.nb_devs++;
 247         }
 248
 249         return cryptodev;
 250 }
 251
 252 static inline int
 253 rte_cryptodev_create_unique_device_name(char *name, size_t size,
 254                 struct rte_pci_device *pci_dev)
 255 {
 256         int ret;
 257
 258         if ((name == NULL) || (pci_dev == NULL))
 259                 return -EINVAL;
 260
 261         ret = snprintf(name, size, "%d:%d.%d",
 262                         pci_dev->addr.bus, pci_dev->addr.devid,
 263                         pci_dev->addr.function);
 264         if (ret < 0)
 265                 return ret;
 266         return 0;
 267 }
 268
 269 int
 270 rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 271 {
 272         int ret;
 273
 274         if (cryptodev == NULL)
 275                 return -EINVAL;
 276
 277         ret = rte_cryptodev_close(cryptodev->data->dev_id);
 278         if (ret < 0)
 279                 return ret;
 280
 281         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 282         cryptodev_globals.nb_devs--;
 283         return 0;
 284 }
 285
 286 struct rte_cryptodev *
 287 rte_cryptodev_pmd_virtual_dev_init(const char *name, size_t dev_private_size,
 288                 int socket_id)
 289 {
 290         struct rte_cryptodev *cryptodev;
 291
 292         /* allocate device structure */
 293         cryptodev = rte_cryptodev_pmd_allocate(name, PMD_VDEV, socket_id);
 294         if (cryptodev == NULL)
 295                 return NULL;
 296
 297         /* allocate private device structure */
 298         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 299                 cryptodev->data->dev_private =
 300                                 rte_zmalloc_socket("cryptodev device private",
 301                                                 dev_private_size,
 302                                                 RTE_CACHE_LINE_SIZE,
 303                                                 socket_id);
 304
 305                 if (cryptodev->data->dev_private == NULL)
 306                         rte_panic("Cannot allocate memzone for private device"
 307                                         " data");
 308         }
 309
 310         /* initialise user call-back tail queue */
 311         TAILQ_INIT(&(cryptodev->link_intr_cbs));
 312
 313         return cryptodev;
 314 }
 315
 316 static int
 317 rte_cryptodev_init(struct rte_pci_driver *pci_drv,
 318                 struct rte_pci_device *pci_dev)
 319 {
 320         struct rte_cryptodev_driver *cryptodrv;
 321         struct rte_cryptodev *cryptodev;
 322
 323         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 324
 325         int retval;
 326
 327         cryptodrv = (struct rte_cryptodev_driver *)pci_drv;
 328         if (cryptodrv == NULL)
 329                 return -ENODEV;
 330
 331         /* Create unique Crypto device name using PCI address */
 332         rte_cryptodev_create_unique_device_name(cryptodev_name,
 333                         sizeof(cryptodev_name), pci_dev);
 334
 335         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, PMD_PDEV,
 336                         rte_socket_id());
 337         if (cryptodev == NULL)
 338                 return -ENOMEM;
 339
 340         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 341                 cryptodev->data->dev_private =
 342                                 rte_zmalloc_socket(
 343                                                 "cryptodev private structure",
 344                                                 cryptodrv->dev_private_size,
 345                                                 RTE_CACHE_LINE_SIZE,
 346                                                 rte_socket_id());
 347
 348                 if (cryptodev->data->dev_private == NULL)
 349                         rte_panic("Cannot allocate memzone for private "
 350                                         "device data");
 351         }
 352
 353         cryptodev->pci_dev = pci_dev;
 354         cryptodev->driver = cryptodrv;
 355
 356         /* init user callbacks */
 357         TAILQ_INIT(&(cryptodev->link_intr_cbs));
 358
 359         /* Invoke PMD device initialization function */
 360         retval = (*cryptodrv->cryptodev_init)(cryptodrv, cryptodev);
 361         if (retval == 0)
 362                 return 0;
 363
 364         CDEV_LOG_ERR("driver %s: crypto_dev_init(vendor_id=0x%x device_id=0x%x)"
 365                         " failed", pci_drv->name,
 366                         (unsigned) pci_dev->id.vendor_id,
 367                         (unsigned) pci_dev->id.device_id);
 368
 369         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 370                 rte_free(cryptodev->data->dev_private);
 371
 372         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 373         cryptodev_globals.nb_devs--;
 374
 375         return -ENXIO;
 376 }
 377
 378 static int
 379 rte_cryptodev_uninit(struct rte_pci_device *pci_dev)
 380 {
 381         const struct rte_cryptodev_driver *cryptodrv;
 382         struct rte_cryptodev *cryptodev;
 383         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 384         int ret;
 385
 386         if (pci_dev == NULL)
 387                 return -EINVAL;
 388
 389         /* Create unique device name using PCI address */
 390         rte_cryptodev_create_unique_device_name(cryptodev_name,
 391                         sizeof(cryptodev_name), pci_dev);
 392
 393         cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
 394         if (cryptodev == NULL)
 395                 return -ENODEV;
 396
 397         cryptodrv = (const struct rte_cryptodev_driver *)pci_dev->driver;
 398         if (cryptodrv == NULL)
 399                 return -ENODEV;
 400
 401         /* Invoke PMD device uninit function */
 402         if (*cryptodrv->cryptodev_uninit) {
 403                 ret = (*cryptodrv->cryptodev_uninit)(cryptodrv, cryptodev);
 404                 if (ret)
 405                         return ret;
 406         }
 407
 408         /* free crypto device */
 409         rte_cryptodev_pmd_release_device(cryptodev);
 410
 411         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 412                 rte_free(cryptodev->data->dev_private);
 413
 414         cryptodev->pci_dev = NULL;
 415         cryptodev->driver = NULL;
 416         cryptodev->data = NULL;
 417
 418         return 0;
 419 }
 420
 421 int
 422 rte_cryptodev_pmd_driver_register(struct rte_cryptodev_driver *cryptodrv,
 423                 enum pmd_type type)
 424 {
 425         /* Call crypto device initialization directly if device is virtual */
 426         if (type == PMD_VDEV)
 427                 return rte_cryptodev_init((struct rte_pci_driver *)cryptodrv,
 428                                 NULL);
 429
 430         /*
 431          * Register PCI driver for physical device intialisation during
 432          * PCI probing
 433          */
 434         cryptodrv->pci_drv.devinit = rte_cryptodev_init;
 435         cryptodrv->pci_drv.devuninit = rte_cryptodev_uninit;
 436
 437         rte_eal_pci_register(&cryptodrv->pci_drv);
 438
 439         return 0;
 440 }
 441
 442
 443 uint16_t
 444 rte_cryptodev_queue_pair_count(uint8_t dev_id)
 445 {
 446         struct rte_cryptodev *dev;
 447
 448         dev = &rte_crypto_devices[dev_id];
 449         return dev->data->nb_queue_pairs;
 450 }
 451
 452 static int
 453 rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 454                 int socket_id)
 455 {
 456         struct rte_cryptodev_info dev_info;
 457         void **qp;
 458         unsigned i;
 459
 460         if ((dev == NULL) || (nb_qpairs < 1)) {
 461                 CDEV_LOG_ERR("invalid param: dev %p, nb_queues %u",
 462                                                         dev, nb_qpairs);
 463                 return -EINVAL;
 464         }
 465
 466         CDEV_LOG_DEBUG("Setup %d queues pairs on device %u",
 467                         nb_qpairs, dev->data->dev_id);
 468
 469         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 470
 471         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 472         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 473
 474         if (nb_qpairs > (dev_info.max_nb_queue_pairs)) {
 475                 CDEV_LOG_ERR("Invalid num queue_pairs (%u) for dev %u",
 476                                 nb_qpairs, dev->data->dev_id);
 477             return -EINVAL;
 478         }
 479
 480         if (dev->data->queue_pairs == NULL) { /* first time configuration */
 481                 dev->data->queue_pairs = rte_zmalloc_socket(
 482                                 "cryptodev->queue_pairs",
 483                                 sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
 484                                 RTE_CACHE_LINE_SIZE, socket_id);
 485
 486                 if (dev->data->queue_pairs == NULL) {
 487                         dev->data->nb_queue_pairs = 0;
 488                         CDEV_LOG_ERR("failed to get memory for qp meta data, "
 489                                                         "nb_queues %u",
 490                                                         nb_qpairs);
 491                         return -(ENOMEM);
 492                 }
 493         } else { /* re-configure */
 494                 int ret;
 495                 uint16_t old_nb_queues = dev->data->nb_queue_pairs;
 496
 497                 qp = dev->data->queue_pairs;
 498
 499                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
 500                                 -ENOTSUP);
 501
 502                 for (i = nb_qpairs; i < old_nb_queues; i++) {
 503                         ret = (*dev->dev_ops->queue_pair_release)(dev, i);
 504                         if (ret < 0)
 505                                 return ret;
 506                 }
 507
 508                 qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
 509                                 RTE_CACHE_LINE_SIZE);
 510                 if (qp == NULL) {
 511                         CDEV_LOG_ERR("failed to realloc qp meta data,"
 512                                                 " nb_queues %u", nb_qpairs);
 513                         return -(ENOMEM);
 514                 }
 515
 516                 if (nb_qpairs > old_nb_queues) {
 517                         uint16_t new_qs = nb_qpairs - old_nb_queues;
 518
 519                         memset(qp + old_nb_queues, 0,
 520                                 sizeof(qp[0]) * new_qs);
 521                 }
 522
 523                 dev->data->queue_pairs = qp;
 524
 525         }
 526         dev->data->nb_queue_pairs = nb_qpairs;
 527         return 0;
 528 }
 529
 530 int
 531 rte_cryptodev_queue_pair_start(uint8_t dev_id, uint16_t queue_pair_id)
 532 {
 533         struct rte_cryptodev *dev;
 534
 535         /*
 536          * This function is only safe when called from the primary process
 537          * in a multi-process setup
 538          */
 539         RTE_PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 540
 541         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 542                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 543                 return -EINVAL;
 544         }
 545
 546         dev = &rte_crypto_devices[dev_id];
 547         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 548                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 549                 return -EINVAL;
 550         }
 551
 552         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_start, -ENOTSUP);
 553
 554         return dev->dev_ops->queue_pair_start(dev, queue_pair_id);
 555
 556 }
 557
 558 int
 559 rte_cryptodev_queue_pair_stop(uint8_t dev_id, uint16_t queue_pair_id)
 560 {
 561         struct rte_cryptodev *dev;
 562
 563         /*
 564          * This function is only safe when called from the primary process
 565          * in a multi-process setup
 566          */
 567         RTE_PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 568
 569         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 570                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 571                 return -EINVAL;
 572         }
 573
 574         dev = &rte_crypto_devices[dev_id];
 575         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 576                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 577                 return -EINVAL;
 578         }
 579
 580         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_stop, -ENOTSUP);
 581
 582         return dev->dev_ops->queue_pair_stop(dev, queue_pair_id);
 583
 584 }
 585
 586 static int
 587 rte_crypto_session_pool_create(struct rte_cryptodev *dev, unsigned nb_objs,
 588                 unsigned obj_cache_size, int socket_id);
 589
 590 int
 591 rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
 592 {
 593         struct rte_cryptodev *dev;
 594         int diag;
 595
 596         /*
 597          * This function is only safe when called from the primary process
 598          * in a multi-process setup
 599          */
 600         RTE_PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 601
 602         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 603                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 604                 return -EINVAL;
 605         }
 606
 607         dev = &rte_crypto_devices[dev_id];
 608
 609         if (dev->data->dev_started) {
 610                 CDEV_LOG_ERR(
 611                     "device %d must be stopped to allow configuration", dev_id);
 612                 return -EBUSY;
 613         }
 614
 615         /* Setup new number of queue pairs and reconfigure device. */
 616         diag = rte_cryptodev_queue_pairs_config(dev, config->nb_queue_pairs,
 617                         config->socket_id);
 618         if (diag != 0) {
 619                 CDEV_LOG_ERR("dev%d rte_crypto_dev_queue_pairs_config = %d",
 620                                 dev_id, diag);
 621                 return diag;
 622         }
 623
 624         /* Setup Session mempool for device */
 625         return rte_crypto_session_pool_create(dev, config->session_mp.nb_objs,
 626                         config->session_mp.cache_size, config->socket_id);
 627 }
 628
 629
 630 int
 631 rte_cryptodev_start(uint8_t dev_id)
 632 {
 633         struct rte_cryptodev *dev;
 634         int diag;
 635
 636         CDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id);
 637
 638         /*
 639          * This function is only safe when called from the primary process
 640          * in a multi-process setup
 641          */
 642         RTE_PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 643
 644         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 645                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 646                 return -EINVAL;
 647         }
 648
 649         dev = &rte_crypto_devices[dev_id];
 650
 651         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
 652
 653         if (dev->data->dev_started != 0) {
 654                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already started",
 655                         dev_id);
 656                 return 0;
 657         }
 658
 659         diag = (*dev->dev_ops->dev_start)(dev);
 660         if (diag == 0)
 661                 dev->data->dev_started = 1;
 662         else
 663                 return diag;
 664
 665         return 0;
 666 }
 667
 668 void
 669 rte_cryptodev_stop(uint8_t dev_id)
 670 {
 671         struct rte_cryptodev *dev;
 672
 673         /*
 674          * This function is only safe when called from the primary process
 675          * in a multi-process setup
 676          */
 677         RTE_PROC_PRIMARY_OR_RET();
 678
 679         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 680                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 681                 return;
 682         }
 683
 684         dev = &rte_crypto_devices[dev_id];
 685
 686         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
 687
 688         if (dev->data->dev_started == 0) {
 689                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already stopped",
 690                         dev_id);
 691                 return;
 692         }
 693
 694         dev->data->dev_started = 0;
 695         (*dev->dev_ops->dev_stop)(dev);
 696 }
 697
 698 int
 699 rte_cryptodev_close(uint8_t dev_id)
 700 {
 701         struct rte_cryptodev *dev;
 702         int retval;
 703
 704         /*
 705          * This function is only safe when called from the primary process
 706          * in a multi-process setup
 707          */
 708         RTE_PROC_PRIMARY_OR_ERR_RET(-EINVAL);
 709
 710         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 711                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 712                 return -1;
 713         }
 714
 715         dev = &rte_crypto_devices[dev_id];
 716
 717         /* Device must be stopped before it can be closed */
 718         if (dev->data->dev_started == 1) {
 719                 CDEV_LOG_ERR("Device %u must be stopped before closing",
 720                                 dev_id);
 721                 return -EBUSY;
 722         }
 723
 724         /* We can't close the device if there are outstanding sessions in use */
 725         if (dev->data->session_pool != NULL) {
 726                 if (!rte_mempool_full(dev->data->session_pool)) {
 727                         CDEV_LOG_ERR("dev_id=%u close failed, session mempool "
 728                                         "has sessions still in use, free "
 729                                         "all sessions before calling close",
 730                                         (unsigned)dev_id);
 731                         return -EBUSY;
 732                 }
 733         }
 734
 735         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
 736         retval = (*dev->dev_ops->dev_close)(dev);
 737
 738         if (retval < 0)
 739                 return retval;
 740
 741         return 0;
 742 }
 743
 744 int
 745 rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
 746                 const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
 747 {
 748         struct rte_cryptodev *dev;
 749
 750         /*
 751          * This function is only safe when called from the primary process
 752          * in a multi-process setup
 753          */
 754         RTE_PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 755
 756         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 757                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 758                 return -EINVAL;
 759         }
 760
 761         dev = &rte_crypto_devices[dev_id];
 762         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 763                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 764                 return -EINVAL;
 765         }
 766
 767         if (dev->data->dev_started) {
 768                 CDEV_LOG_ERR(
 769                     "device %d must be stopped to allow configuration", dev_id);
 770                 return -EBUSY;
 771         }
 772
 773         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
 774
 775         return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id, qp_conf,
 776                         socket_id);
 777 }
 778
 779
 780 int
 781 rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats)
 782 {
 783         struct rte_cryptodev *dev;
 784
 785         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 786                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
 787                 return -ENODEV;
 788         }
 789
 790         if (stats == NULL) {
 791                 CDEV_LOG_ERR("Invalid stats ptr");
 792                 return -EINVAL;
 793         }
 794
 795         dev = &rte_crypto_devices[dev_id];
 796         memset(stats, 0, sizeof(*stats));
 797
 798         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
 799         (*dev->dev_ops->stats_get)(dev, stats);
 800         return 0;
 801 }
 802
 803 void
 804 rte_cryptodev_stats_reset(uint8_t dev_id)
 805 {
 806         struct rte_cryptodev *dev;
 807
 808         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 809                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 810                 return;
 811         }
 812
 813         dev = &rte_crypto_devices[dev_id];
 814
 815         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
 816         (*dev->dev_ops->stats_reset)(dev);
 817 }
 818
 819
 820 void
 821 rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
 822 {
 823         struct rte_cryptodev *dev;
 824
 825         if (dev_id >= cryptodev_globals.nb_devs) {
 826                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
 827                 return;
 828         }
 829
 830         dev = &rte_crypto_devices[dev_id];
 831
 832         memset(dev_info, 0, sizeof(struct rte_cryptodev_info));
 833
 834         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
 835         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
 836
 837         dev_info->pci_dev = dev->pci_dev;
 838         if (dev->driver)
 839                 dev_info->driver_name = dev->driver->pci_drv.name;
 840 }
 841
 842
 843 int
 844 rte_cryptodev_callback_register(uint8_t dev_id,
 845                         enum rte_cryptodev_event_type event,
 846                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
 847 {
 848         struct rte_cryptodev *dev;
 849         struct rte_cryptodev_callback *user_cb;
 850
 851         if (!cb_fn)
 852                 return -EINVAL;
 853
 854         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 855                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 856                 return -EINVAL;
 857         }
 858
 859         dev = &rte_crypto_devices[dev_id];
 860         rte_spinlock_lock(&rte_cryptodev_cb_lock);
 861
 862         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
 863                 if (user_cb->cb_fn == cb_fn &&
 864                         user_cb->cb_arg == cb_arg &&
 865                         user_cb->event == event) {
 866                         break;
 867                 }
 868         }
 869
 870         /* create a new callback. */
 871         if (user_cb == NULL) {
 872                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
 873                                 sizeof(struct rte_cryptodev_callback), 0);
 874                 if (user_cb != NULL) {
 875                         user_cb->cb_fn = cb_fn;
 876                         user_cb->cb_arg = cb_arg;
 877                         user_cb->event = event;
 878                         TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
 879                 }
 880         }
 881
 882         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 883         return (user_cb == NULL) ? -ENOMEM : 0;
 884 }
 885
 886 int
 887 rte_cryptodev_callback_unregister(uint8_t dev_id,
 888                         enum rte_cryptodev_event_type event,
 889                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
 890 {
 891         int ret;
 892         struct rte_cryptodev *dev;
 893         struct rte_cryptodev_callback *cb, *next;
 894
 895         if (!cb_fn)
 896                 return -EINVAL;
 897
 898         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 899                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 900                 return -EINVAL;
 901         }
 902
 903         dev = &rte_crypto_devices[dev_id];
 904         rte_spinlock_lock(&rte_cryptodev_cb_lock);
 905
 906         ret = 0;
 907         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
 908
 909                 next = TAILQ_NEXT(cb, next);
 910
 911                 if (cb->cb_fn != cb_fn || cb->event != event ||
 912                                 (cb->cb_arg != (void *)-1 &&
 913                                 cb->cb_arg != cb_arg))
 914                         continue;
 915
 916                 /*
 917                  * if this callback is not executing right now,
 918                  * then remove it.
 919                  */
 920                 if (cb->active == 0) {
 921                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
 922                         rte_free(cb);
 923                 } else {
 924                         ret = -EAGAIN;
 925                 }
 926         }
 927
 928         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 929         return ret;
 930 }
 931
 932 void
 933 rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev,
 934         enum rte_cryptodev_event_type event)
 935 {
 936         struct rte_cryptodev_callback *cb_lst;
 937         struct rte_cryptodev_callback dev_cb;
 938
 939         rte_spinlock_lock(&rte_cryptodev_cb_lock);
 940         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
 941                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
 942                         continue;
 943                 dev_cb = *cb_lst;
 944                 cb_lst->active = 1;
 945                 rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 946                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
 947                                                 dev_cb.cb_arg);
 948                 rte_spinlock_lock(&rte_cryptodev_cb_lock);
 949                 cb_lst->active = 0;
 950         }
 951         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 952 }
 953
 954
 955 static void
 956 rte_crypto_session_init(struct rte_mempool *mp,
 957                 void *opaque_arg,
 958                 void *_sess,
 959                 __rte_unused unsigned i)
 960 {
 961         struct rte_cryptodev_session *sess = _sess;
 962         struct rte_cryptodev *dev = opaque_arg;
 963
 964         memset(sess, 0, mp->elt_size);
 965
 966         sess->dev_id = dev->data->dev_id;
 967         sess->type = dev->dev_type;
 968         sess->mp = mp;
 969
 970         if (dev->dev_ops->session_initialize)
 971                 (*dev->dev_ops->session_initialize)(mp, sess->_private);
 972 }
 973
 974 static int
 975 rte_crypto_session_pool_create(struct rte_cryptodev *dev, unsigned nb_objs,
 976                 unsigned obj_cache_size, int socket_id)
 977 {
 978         char mp_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 979         unsigned priv_sess_size;
 980
 981         unsigned n = snprintf(mp_name, sizeof(mp_name), "cdev_%d_sess_mp",
 982                         dev->data->dev_id);
 983         if (n > sizeof(mp_name)) {
 984                 CDEV_LOG_ERR("Unable to create unique name for session mempool");
 985                 return -ENOMEM;
 986         }
 987
 988         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_get_size, -ENOTSUP);
 989         priv_sess_size = (*dev->dev_ops->session_get_size)(dev);
 990         if (priv_sess_size == 0) {
 991                 CDEV_LOG_ERR("%s returned and invalid private session size ",
 992                                                 dev->data->name);
 993                 return -ENOMEM;
 994         }
 995
 996         unsigned elt_size = sizeof(struct rte_cryptodev_session) +
 997                         priv_sess_size;
 998
 999         dev->data->session_pool = rte_mempool_lookup(mp_name);
1000         if (dev->data->session_pool != NULL) {
1001                 if ((dev->data->session_pool->elt_size != elt_size) ||
1002                                 (dev->data->session_pool->cache_size <
1003                                 obj_cache_size) ||
1004                                 (dev->data->session_pool->size < nb_objs)) {
1005
1006                         CDEV_LOG_ERR("%s mempool already exists with different"
1007                                         " initialization parameters", mp_name);
1008                         dev->data->session_pool = NULL;
1009                         return -ENOMEM;
1010                 }
1011         } else {
1012                 dev->data->session_pool = rte_mempool_create(
1013                                 mp_name, /* mempool name */
1014                                 nb_objs, /* number of elements*/
1015                                 elt_size, /* element size*/
1016                                 obj_cache_size, /* Cache size*/
1017                                 0, /* private data size */
1018                                 NULL, /* obj initialization constructor */
1019                                 NULL, /* obj initialization constructor arg */
1020                                 rte_crypto_session_init, /* obj constructor */
1021                                 dev, /* obj constructor arg */
1022                                 socket_id, /* socket id */
1023                                 0); /* flags */
1024
1025                 if (dev->data->session_pool == NULL) {
1026                         CDEV_LOG_ERR("%s mempool allocation failed", mp_name);
1027                         return -ENOMEM;
1028                 }
1029         }
1030
1031         CDEV_LOG_DEBUG("%s mempool created!", mp_name);
1032         return 0;
1033 }
1034
1035 struct rte_cryptodev_session *
1036 rte_cryptodev_session_create(uint8_t dev_id, struct rte_crypto_xform *xform)
1037 {
1038         struct rte_cryptodev *dev;
1039         struct rte_cryptodev_session *sess;
1040         void *_sess;
1041
1042         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1043                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1044                 return NULL;
1045         }
1046
1047         dev = &rte_crypto_devices[dev_id];
1048
1049         /* Allocate a session structure from the session pool */
1050         if (rte_mempool_get(dev->data->session_pool, &_sess)) {
1051                 CDEV_LOG_ERR("Couldn't get object from session mempool");
1052                 return NULL;
1053         }
1054
1055         sess = (struct rte_cryptodev_session *)_sess;
1056
1057         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_configure, NULL);
1058         if (dev->dev_ops->session_configure(dev, xform, sess->_private) ==
1059                         NULL) {
1060                 CDEV_LOG_ERR("dev_id %d failed to configure session details",
1061                                 dev_id);
1062
1063                 /* Return session to mempool */
1064                 rte_mempool_put(sess->mp, _sess);
1065                 return NULL;
1066         }
1067
1068         return sess;
1069 }
1070
1071 struct rte_cryptodev_session *
1072 rte_cryptodev_session_free(uint8_t dev_id, struct rte_cryptodev_session *sess)
1073 {
1074         struct rte_cryptodev *dev;
1075
1076         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1077                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1078                 return sess;
1079         }
1080
1081         dev = &rte_crypto_devices[dev_id];
1082
1083         /* Check the session belongs to this device type */
1084         if (sess->type != dev->dev_type)
1085                 return sess;
1086
1087         /* Let device implementation clear session material */
1088         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_clear, sess);
1089         dev->dev_ops->session_clear(dev, (void *)sess->_private);
1090
1091         /* Return session to mempool */
1092         rte_mempool_put(sess->mp, (void *)sess);
1093
1094         return NULL;
1095 }