lib/cryptodev/rte_cryptodev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2020 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdint.h>
  14 #include <inttypes.h>
  15 #include <netinet/in.h>
  16
  17 #include <rte_byteorder.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <rte_dev.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_memory.h>
  23 #include <rte_memcpy.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_tailq.h>
  27 #include <rte_eal.h>
  28 #include <rte_per_lcore.h>
  29 #include <rte_lcore.h>
  30 #include <rte_atomic.h>
  31 #include <rte_branch_prediction.h>
  32 #include <rte_common.h>
  33 #include <rte_mempool.h>
  34 #include <rte_malloc.h>
  35 #include <rte_mbuf.h>
  36 #include <rte_errno.h>
  37 #include <rte_spinlock.h>
  38 #include <rte_string_fns.h>
  39
  40 #include "rte_crypto.h"
  41 #include "rte_cryptodev.h"
  42 #include "cryptodev_pmd.h"
  43 #include "rte_cryptodev_trace.h"
  44
  45 static uint8_t nb_drivers;
  46
  47 static struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
  48
  49 struct rte_cryptodev *rte_cryptodevs = rte_crypto_devices;
  50
  51 static struct rte_cryptodev_global cryptodev_globals = {
  52                 .devs                   = rte_crypto_devices,
  53                 .data                   = { NULL },
  54                 .nb_devs                = 0
  55 };
  56
  57 /* spinlock for crypto device callbacks */
  58 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  59
  60 /**
  61  * The user application callback description.
  62  *
  63  * It contains callback address to be registered by user application,
  64  * the pointer to the parameters for callback, and the event type.
  65  */
  66 struct rte_cryptodev_callback {
  67         TAILQ_ENTRY(rte_cryptodev_callback) next; /**< Callbacks list */
  68         rte_cryptodev_cb_fn cb_fn;              /**< Callback address */
  69         void *cb_arg;                           /**< Parameter for callback */
  70         enum rte_cryptodev_event_type event;    /**< Interrupt event type */
  71         uint32_t active;                        /**< Callback is executing */
  72 };
  73
  74 /**
  75  * The crypto cipher algorithm strings identifiers.
  76  * It could be used in application command line.
  77  */
  78 const char *
  79 rte_crypto_cipher_algorithm_strings[] = {
  80         [RTE_CRYPTO_CIPHER_3DES_CBC]    = "3des-cbc",
  81         [RTE_CRYPTO_CIPHER_3DES_ECB]    = "3des-ecb",
  82         [RTE_CRYPTO_CIPHER_3DES_CTR]    = "3des-ctr",
  83
  84         [RTE_CRYPTO_CIPHER_AES_CBC]     = "aes-cbc",
  85         [RTE_CRYPTO_CIPHER_AES_CTR]     = "aes-ctr",
  86         [RTE_CRYPTO_CIPHER_AES_DOCSISBPI]       = "aes-docsisbpi",
  87         [RTE_CRYPTO_CIPHER_AES_ECB]     = "aes-ecb",
  88         [RTE_CRYPTO_CIPHER_AES_F8]      = "aes-f8",
  89         [RTE_CRYPTO_CIPHER_AES_XTS]     = "aes-xts",
  90
  91         [RTE_CRYPTO_CIPHER_ARC4]        = "arc4",
  92
  93         [RTE_CRYPTO_CIPHER_DES_CBC]     = "des-cbc",
  94         [RTE_CRYPTO_CIPHER_DES_DOCSISBPI]       = "des-docsisbpi",
  95
  96         [RTE_CRYPTO_CIPHER_NULL]        = "null",
  97
  98         [RTE_CRYPTO_CIPHER_KASUMI_F8]   = "kasumi-f8",
  99         [RTE_CRYPTO_CIPHER_SNOW3G_UEA2] = "snow3g-uea2",
 100         [RTE_CRYPTO_CIPHER_ZUC_EEA3]    = "zuc-eea3"
 101 };
 102
 103 /**
 104  * The crypto cipher operation strings identifiers.
 105  * It could be used in application command line.
 106  */
 107 const char *
 108 rte_crypto_cipher_operation_strings[] = {
 109                 [RTE_CRYPTO_CIPHER_OP_ENCRYPT]  = "encrypt",
 110                 [RTE_CRYPTO_CIPHER_OP_DECRYPT]  = "decrypt"
 111 };
 112
 113 /**
 114  * The crypto auth algorithm strings identifiers.
 115  * It could be used in application command line.
 116  */
 117 const char *
 118 rte_crypto_auth_algorithm_strings[] = {
 119         [RTE_CRYPTO_AUTH_AES_CBC_MAC]   = "aes-cbc-mac",
 120         [RTE_CRYPTO_AUTH_AES_CMAC]      = "aes-cmac",
 121         [RTE_CRYPTO_AUTH_AES_GMAC]      = "aes-gmac",
 122         [RTE_CRYPTO_AUTH_AES_XCBC_MAC]  = "aes-xcbc-mac",
 123
 124         [RTE_CRYPTO_AUTH_MD5]           = "md5",
 125         [RTE_CRYPTO_AUTH_MD5_HMAC]      = "md5-hmac",
 126
 127         [RTE_CRYPTO_AUTH_NULL]          = "null",
 128
 129         [RTE_CRYPTO_AUTH_SHA1]          = "sha1",
 130         [RTE_CRYPTO_AUTH_SHA1_HMAC]     = "sha1-hmac",
 131
 132         [RTE_CRYPTO_AUTH_SHA224]        = "sha2-224",
 133         [RTE_CRYPTO_AUTH_SHA224_HMAC]   = "sha2-224-hmac",
 134         [RTE_CRYPTO_AUTH_SHA256]        = "sha2-256",
 135         [RTE_CRYPTO_AUTH_SHA256_HMAC]   = "sha2-256-hmac",
 136         [RTE_CRYPTO_AUTH_SHA384]        = "sha2-384",
 137         [RTE_CRYPTO_AUTH_SHA384_HMAC]   = "sha2-384-hmac",
 138         [RTE_CRYPTO_AUTH_SHA512]        = "sha2-512",
 139         [RTE_CRYPTO_AUTH_SHA512_HMAC]   = "sha2-512-hmac",
 140
 141         [RTE_CRYPTO_AUTH_KASUMI_F9]     = "kasumi-f9",
 142         [RTE_CRYPTO_AUTH_SNOW3G_UIA2]   = "snow3g-uia2",
 143         [RTE_CRYPTO_AUTH_ZUC_EIA3]      = "zuc-eia3"
 144 };
 145
 146 /**
 147  * The crypto AEAD algorithm strings identifiers.
 148  * It could be used in application command line.
 149  */
 150 const char *
 151 rte_crypto_aead_algorithm_strings[] = {
 152         [RTE_CRYPTO_AEAD_AES_CCM]       = "aes-ccm",
 153         [RTE_CRYPTO_AEAD_AES_GCM]       = "aes-gcm",
 154         [RTE_CRYPTO_AEAD_CHACHA20_POLY1305] = "chacha20-poly1305"
 155 };
 156
 157 /**
 158  * The crypto AEAD operation strings identifiers.
 159  * It could be used in application command line.
 160  */
 161 const char *
 162 rte_crypto_aead_operation_strings[] = {
 163         [RTE_CRYPTO_AEAD_OP_ENCRYPT]    = "encrypt",
 164         [RTE_CRYPTO_AEAD_OP_DECRYPT]    = "decrypt"
 165 };
 166
 167 /**
 168  * Asymmetric crypto transform operation strings identifiers.
 169  */
 170 const char *rte_crypto_asym_xform_strings[] = {
 171         [RTE_CRYPTO_ASYM_XFORM_NONE]    = "none",
 172         [RTE_CRYPTO_ASYM_XFORM_RSA]     = "rsa",
 173         [RTE_CRYPTO_ASYM_XFORM_MODEX]   = "modexp",
 174         [RTE_CRYPTO_ASYM_XFORM_MODINV]  = "modinv",
 175         [RTE_CRYPTO_ASYM_XFORM_DH]      = "dh",
 176         [RTE_CRYPTO_ASYM_XFORM_DSA]     = "dsa",
 177         [RTE_CRYPTO_ASYM_XFORM_ECDSA]   = "ecdsa",
 178         [RTE_CRYPTO_ASYM_XFORM_ECPM]    = "ecpm",
 179 };
 180
 181 /**
 182  * Asymmetric crypto operation strings identifiers.
 183  */
 184 const char *rte_crypto_asym_op_strings[] = {
 185         [RTE_CRYPTO_ASYM_OP_ENCRYPT]    = "encrypt",
 186         [RTE_CRYPTO_ASYM_OP_DECRYPT]    = "decrypt",
 187         [RTE_CRYPTO_ASYM_OP_SIGN]       = "sign",
 188         [RTE_CRYPTO_ASYM_OP_VERIFY]     = "verify",
 189         [RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE]       = "priv_key_generate",
 190         [RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE] = "pub_key_generate",
 191         [RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE] = "sharedsecret_compute",
 192 };
 193
 194 /**
 195  * The private data structure stored in the session mempool private data.
 196  */
 197 struct rte_cryptodev_sym_session_pool_private_data {
 198         uint16_t nb_drivers;
 199         /**< number of elements in sess_data array */
 200         uint16_t user_data_sz;
 201         /**< session user data will be placed after sess_data */
 202 };
 203
 204 int
 205 rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum,
 206                 const char *algo_string)
 207 {
 208         unsigned int i;
 209
 210         for (i = 1; i < RTE_DIM(rte_crypto_cipher_algorithm_strings); i++) {
 211                 if (strcmp(algo_string, rte_crypto_cipher_algorithm_strings[i]) == 0) {
 212                         *algo_enum = (enum rte_crypto_cipher_algorithm) i;
 213                         return 0;
 214                 }
 215         }
 216
 217         /* Invalid string */
 218         return -1;
 219 }
 220
 221 int
 222 rte_cryptodev_get_auth_algo_enum(enum rte_crypto_auth_algorithm *algo_enum,
 223                 const char *algo_string)
 224 {
 225         unsigned int i;
 226
 227         for (i = 1; i < RTE_DIM(rte_crypto_auth_algorithm_strings); i++) {
 228                 if (strcmp(algo_string, rte_crypto_auth_algorithm_strings[i]) == 0) {
 229                         *algo_enum = (enum rte_crypto_auth_algorithm) i;
 230                         return 0;
 231                 }
 232         }
 233
 234         /* Invalid string */
 235         return -1;
 236 }
 237
 238 int
 239 rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum,
 240                 const char *algo_string)
 241 {
 242         unsigned int i;
 243
 244         for (i = 1; i < RTE_DIM(rte_crypto_aead_algorithm_strings); i++) {
 245                 if (strcmp(algo_string, rte_crypto_aead_algorithm_strings[i]) == 0) {
 246                         *algo_enum = (enum rte_crypto_aead_algorithm) i;
 247                         return 0;
 248                 }
 249         }
 250
 251         /* Invalid string */
 252         return -1;
 253 }
 254
 255 int
 256 rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
 257                 const char *xform_string)
 258 {
 259         unsigned int i;
 260
 261         for (i = 1; i < RTE_DIM(rte_crypto_asym_xform_strings); i++) {
 262                 if (strcmp(xform_string,
 263                         rte_crypto_asym_xform_strings[i]) == 0) {
 264                         *xform_enum = (enum rte_crypto_asym_xform_type) i;
 265                         return 0;
 266                 }
 267         }
 268
 269         /* Invalid string */
 270         return -1;
 271 }
 272
 273 /**
 274  * The crypto auth operation strings identifiers.
 275  * It could be used in application command line.
 276  */
 277 const char *
 278 rte_crypto_auth_operation_strings[] = {
 279                 [RTE_CRYPTO_AUTH_OP_VERIFY]     = "verify",
 280                 [RTE_CRYPTO_AUTH_OP_GENERATE]   = "generate"
 281 };
 282
 283 const struct rte_cryptodev_symmetric_capability *
 284 rte_cryptodev_sym_capability_get(uint8_t dev_id,
 285                 const struct rte_cryptodev_sym_capability_idx *idx)
 286 {
 287         const struct rte_cryptodev_capabilities *capability;
 288         struct rte_cryptodev_info dev_info;
 289         int i = 0;
 290
 291         rte_cryptodev_info_get(dev_id, &dev_info);
 292
 293         while ((capability = &dev_info.capabilities[i++])->op !=
 294                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 295                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
 296                         continue;
 297
 298                 if (capability->sym.xform_type != idx->type)
 299                         continue;
 300
 301                 if (idx->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 302                         capability->sym.auth.algo == idx->algo.auth)
 303                         return &capability->sym;
 304
 305                 if (idx->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 306                         capability->sym.cipher.algo == idx->algo.cipher)
 307                         return &capability->sym;
 308
 309                 if (idx->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
 310                                 capability->sym.aead.algo == idx->algo.aead)
 311                         return &capability->sym;
 312         }
 313
 314         return NULL;
 315 }
 316
 317 static int
 318 param_range_check(uint16_t size, const struct rte_crypto_param_range *range)
 319 {
 320         unsigned int next_size;
 321
 322         /* Check lower/upper bounds */
 323         if (size < range->min)
 324                 return -1;
 325
 326         if (size > range->max)
 327                 return -1;
 328
 329         /* If range is actually only one value, size is correct */
 330         if (range->increment == 0)
 331                 return 0;
 332
 333         /* Check if value is one of the supported sizes */
 334         for (next_size = range->min; next_size <= range->max;
 335                         next_size += range->increment)
 336                 if (size == next_size)
 337                         return 0;
 338
 339         return -1;
 340 }
 341
 342 const struct rte_cryptodev_asymmetric_xform_capability *
 343 rte_cryptodev_asym_capability_get(uint8_t dev_id,
 344                 const struct rte_cryptodev_asym_capability_idx *idx)
 345 {
 346         const struct rte_cryptodev_capabilities *capability;
 347         struct rte_cryptodev_info dev_info;
 348         unsigned int i = 0;
 349
 350         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 351         rte_cryptodev_info_get(dev_id, &dev_info);
 352
 353         while ((capability = &dev_info.capabilities[i++])->op !=
 354                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 355                 if (capability->op != RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
 356                         continue;
 357
 358                 if (capability->asym.xform_capa.xform_type == idx->type)
 359                         return &capability->asym.xform_capa;
 360         }
 361         return NULL;
 362 };
 363
 364 int
 365 rte_cryptodev_sym_capability_check_cipher(
 366                 const struct rte_cryptodev_symmetric_capability *capability,
 367                 uint16_t key_size, uint16_t iv_size)
 368 {
 369         if (param_range_check(key_size, &capability->cipher.key_size) != 0)
 370                 return -1;
 371
 372         if (param_range_check(iv_size, &capability->cipher.iv_size) != 0)
 373                 return -1;
 374
 375         return 0;
 376 }
 377
 378 int
 379 rte_cryptodev_sym_capability_check_auth(
 380                 const struct rte_cryptodev_symmetric_capability *capability,
 381                 uint16_t key_size, uint16_t digest_size, uint16_t iv_size)
 382 {
 383         if (param_range_check(key_size, &capability->auth.key_size) != 0)
 384                 return -1;
 385
 386         if (param_range_check(digest_size, &capability->auth.digest_size) != 0)
 387                 return -1;
 388
 389         if (param_range_check(iv_size, &capability->auth.iv_size) != 0)
 390                 return -1;
 391
 392         return 0;
 393 }
 394
 395 int
 396 rte_cryptodev_sym_capability_check_aead(
 397                 const struct rte_cryptodev_symmetric_capability *capability,
 398                 uint16_t key_size, uint16_t digest_size, uint16_t aad_size,
 399                 uint16_t iv_size)
 400 {
 401         if (param_range_check(key_size, &capability->aead.key_size) != 0)
 402                 return -1;
 403
 404         if (param_range_check(digest_size, &capability->aead.digest_size) != 0)
 405                 return -1;
 406
 407         if (param_range_check(aad_size, &capability->aead.aad_size) != 0)
 408                 return -1;
 409
 410         if (param_range_check(iv_size, &capability->aead.iv_size) != 0)
 411                 return -1;
 412
 413         return 0;
 414 }
 415 int
 416 rte_cryptodev_asym_xform_capability_check_optype(
 417         const struct rte_cryptodev_asymmetric_xform_capability *capability,
 418         enum rte_crypto_asym_op_type op_type)
 419 {
 420         if (capability->op_types & (1 << op_type))
 421                 return 1;
 422
 423         return 0;
 424 }
 425
 426 int
 427 rte_cryptodev_asym_xform_capability_check_modlen(
 428         const struct rte_cryptodev_asymmetric_xform_capability *capability,
 429         uint16_t modlen)
 430 {
 431         /* no need to check for limits, if min or max = 0 */
 432         if (capability->modlen.min != 0) {
 433                 if (modlen < capability->modlen.min)
 434                         return -1;
 435         }
 436
 437         if (capability->modlen.max != 0) {
 438                 if (modlen > capability->modlen.max)
 439                         return -1;
 440         }
 441
 442         /* in any case, check if given modlen is module increment */
 443         if (capability->modlen.increment != 0) {
 444                 if (modlen % (capability->modlen.increment))
 445                         return -1;
 446         }
 447
 448         return 0;
 449 }
 450
 451 /* spinlock for crypto device enq callbacks */
 452 static rte_spinlock_t rte_cryptodev_callback_lock = RTE_SPINLOCK_INITIALIZER;
 453
 454 static void
 455 cryptodev_cb_cleanup(struct rte_cryptodev *dev)
 456 {
 457         struct rte_cryptodev_cb_rcu *list;
 458         struct rte_cryptodev_cb *cb, *next;
 459         uint16_t qp_id;
 460
 461         if (dev->enq_cbs == NULL && dev->deq_cbs == NULL)
 462                 return;
 463
 464         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 465                 list = &dev->enq_cbs[qp_id];
 466                 cb = list->next;
 467                 while (cb != NULL) {
 468                         next = cb->next;
 469                         rte_free(cb);
 470                         cb = next;
 471                 }
 472
 473                 rte_free(list->qsbr);
 474         }
 475
 476         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 477                 list = &dev->deq_cbs[qp_id];
 478                 cb = list->next;
 479                 while (cb != NULL) {
 480                         next = cb->next;
 481                         rte_free(cb);
 482                         cb = next;
 483                 }
 484
 485                 rte_free(list->qsbr);
 486         }
 487
 488         rte_free(dev->enq_cbs);
 489         dev->enq_cbs = NULL;
 490         rte_free(dev->deq_cbs);
 491         dev->deq_cbs = NULL;
 492 }
 493
 494 static int
 495 cryptodev_cb_init(struct rte_cryptodev *dev)
 496 {
 497         struct rte_cryptodev_cb_rcu *list;
 498         struct rte_rcu_qsbr *qsbr;
 499         uint16_t qp_id;
 500         size_t size;
 501
 502         /* Max thread set to 1, as one DP thread accessing a queue-pair */
 503         const uint32_t max_threads = 1;
 504
 505         dev->enq_cbs = rte_zmalloc(NULL,
 506                                    sizeof(struct rte_cryptodev_cb_rcu) *
 507                                    dev->data->nb_queue_pairs, 0);
 508         if (dev->enq_cbs == NULL) {
 509                 CDEV_LOG_ERR("Failed to allocate memory for enq callbacks");
 510                 return -ENOMEM;
 511         }
 512
 513         dev->deq_cbs = rte_zmalloc(NULL,
 514                                    sizeof(struct rte_cryptodev_cb_rcu) *
 515                                    dev->data->nb_queue_pairs, 0);
 516         if (dev->deq_cbs == NULL) {
 517                 CDEV_LOG_ERR("Failed to allocate memory for deq callbacks");
 518                 rte_free(dev->enq_cbs);
 519                 return -ENOMEM;
 520         }
 521
 522         /* Create RCU QSBR variable */
 523         size = rte_rcu_qsbr_get_memsize(max_threads);
 524
 525         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 526                 list = &dev->enq_cbs[qp_id];
 527                 qsbr = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 528                 if (qsbr == NULL) {
 529                         CDEV_LOG_ERR("Failed to allocate memory for RCU on "
 530                                 "queue_pair_id=%d", qp_id);
 531                         goto cb_init_err;
 532                 }
 533
 534                 if (rte_rcu_qsbr_init(qsbr, max_threads)) {
 535                         CDEV_LOG_ERR("Failed to initialize for RCU on "
 536                                 "queue_pair_id=%d", qp_id);
 537                         goto cb_init_err;
 538                 }
 539
 540                 list->qsbr = qsbr;
 541         }
 542
 543         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 544                 list = &dev->deq_cbs[qp_id];
 545                 qsbr = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 546                 if (qsbr == NULL) {
 547                         CDEV_LOG_ERR("Failed to allocate memory for RCU on "
 548                                 "queue_pair_id=%d", qp_id);
 549                         goto cb_init_err;
 550                 }
 551
 552                 if (rte_rcu_qsbr_init(qsbr, max_threads)) {
 553                         CDEV_LOG_ERR("Failed to initialize for RCU on "
 554                                 "queue_pair_id=%d", qp_id);
 555                         goto cb_init_err;
 556                 }
 557
 558                 list->qsbr = qsbr;
 559         }
 560
 561         return 0;
 562
 563 cb_init_err:
 564         cryptodev_cb_cleanup(dev);
 565         return -ENOMEM;
 566 }
 567
 568 const char *
 569 rte_cryptodev_get_feature_name(uint64_t flag)
 570 {
 571         switch (flag) {
 572         case RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO:
 573                 return "SYMMETRIC_CRYPTO";
 574         case RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO:
 575                 return "ASYMMETRIC_CRYPTO";
 576         case RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING:
 577                 return "SYM_OPERATION_CHAINING";
 578         case RTE_CRYPTODEV_FF_CPU_SSE:
 579                 return "CPU_SSE";
 580         case RTE_CRYPTODEV_FF_CPU_AVX:
 581                 return "CPU_AVX";
 582         case RTE_CRYPTODEV_FF_CPU_AVX2:
 583                 return "CPU_AVX2";
 584         case RTE_CRYPTODEV_FF_CPU_AVX512:
 585                 return "CPU_AVX512";
 586         case RTE_CRYPTODEV_FF_CPU_AESNI:
 587                 return "CPU_AESNI";
 588         case RTE_CRYPTODEV_FF_HW_ACCELERATED:
 589                 return "HW_ACCELERATED";
 590         case RTE_CRYPTODEV_FF_IN_PLACE_SGL:
 591                 return "IN_PLACE_SGL";
 592         case RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT:
 593                 return "OOP_SGL_IN_SGL_OUT";
 594         case RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT:
 595                 return "OOP_SGL_IN_LB_OUT";
 596         case RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT:
 597                 return "OOP_LB_IN_SGL_OUT";
 598         case RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT:
 599                 return "OOP_LB_IN_LB_OUT";
 600         case RTE_CRYPTODEV_FF_CPU_NEON:
 601                 return "CPU_NEON";
 602         case RTE_CRYPTODEV_FF_CPU_ARM_CE:
 603                 return "CPU_ARM_CE";
 604         case RTE_CRYPTODEV_FF_SECURITY:
 605                 return "SECURITY_PROTOCOL";
 606         case RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP:
 607                 return "RSA_PRIV_OP_KEY_EXP";
 608         case RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT:
 609                 return "RSA_PRIV_OP_KEY_QT";
 610         case RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED:
 611                 return "DIGEST_ENCRYPTED";
 612         case RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO:
 613                 return "SYM_CPU_CRYPTO";
 614         case RTE_CRYPTODEV_FF_ASYM_SESSIONLESS:
 615                 return "ASYM_SESSIONLESS";
 616         case RTE_CRYPTODEV_FF_SYM_SESSIONLESS:
 617                 return "SYM_SESSIONLESS";
 618         case RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA:
 619                 return "NON_BYTE_ALIGNED_DATA";
 620         case RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS:
 621                 return "CIPHER_MULTIPLE_DATA_UNITS";
 622         case RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY:
 623                 return "CIPHER_WRAPPED_KEY";
 624         default:
 625                 return NULL;
 626         }
 627 }
 628
 629 struct rte_cryptodev *
 630 rte_cryptodev_pmd_get_dev(uint8_t dev_id)
 631 {
 632         return &cryptodev_globals.devs[dev_id];
 633 }
 634
 635 struct rte_cryptodev *
 636 rte_cryptodev_pmd_get_named_dev(const char *name)
 637 {
 638         struct rte_cryptodev *dev;
 639         unsigned int i;
 640
 641         if (name == NULL)
 642                 return NULL;
 643
 644         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
 645                 dev = &cryptodev_globals.devs[i];
 646
 647                 if ((dev->attached == RTE_CRYPTODEV_ATTACHED) &&
 648                                 (strcmp(dev->data->name, name) == 0))
 649                         return dev;
 650         }
 651
 652         return NULL;
 653 }
 654
 655 static inline uint8_t
 656 rte_cryptodev_is_valid_device_data(uint8_t dev_id)
 657 {
 658         if (dev_id >= RTE_CRYPTO_MAX_DEVS ||
 659                         rte_crypto_devices[dev_id].data == NULL)
 660                 return 0;
 661
 662         return 1;
 663 }
 664
 665 unsigned int
 666 rte_cryptodev_is_valid_dev(uint8_t dev_id)
 667 {
 668         struct rte_cryptodev *dev = NULL;
 669
 670         if (!rte_cryptodev_is_valid_device_data(dev_id))
 671                 return 0;
 672
 673         dev = rte_cryptodev_pmd_get_dev(dev_id);
 674         if (dev->attached != RTE_CRYPTODEV_ATTACHED)
 675                 return 0;
 676         else
 677                 return 1;
 678 }
 679
 680
 681 int
 682 rte_cryptodev_get_dev_id(const char *name)
 683 {
 684         unsigned i;
 685
 686         if (name == NULL)
 687                 return -1;
 688
 689         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
 690                 if (!rte_cryptodev_is_valid_device_data(i))
 691                         continue;
 692                 if ((strcmp(cryptodev_globals.devs[i].data->name, name)
 693                                 == 0) &&
 694                                 (cryptodev_globals.devs[i].attached ==
 695                                                 RTE_CRYPTODEV_ATTACHED))
 696                         return i;
 697         }
 698
 699         return -1;
 700 }
 701
 702 uint8_t
 703 rte_cryptodev_count(void)
 704 {
 705         return cryptodev_globals.nb_devs;
 706 }
 707
 708 uint8_t
 709 rte_cryptodev_device_count_by_driver(uint8_t driver_id)
 710 {
 711         uint8_t i, dev_count = 0;
 712
 713         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++)
 714                 if (cryptodev_globals.devs[i].driver_id == driver_id &&
 715                         cryptodev_globals.devs[i].attached ==
 716                                         RTE_CRYPTODEV_ATTACHED)
 717                         dev_count++;
 718
 719         return dev_count;
 720 }
 721
 722 uint8_t
 723 rte_cryptodev_devices_get(const char *driver_name, uint8_t *devices,
 724         uint8_t nb_devices)
 725 {
 726         uint8_t i, count = 0;
 727         struct rte_cryptodev *devs = cryptodev_globals.devs;
 728
 729         for (i = 0; i < RTE_CRYPTO_MAX_DEVS && count < nb_devices; i++) {
 730                 if (!rte_cryptodev_is_valid_device_data(i))
 731                         continue;
 732
 733                 if (devs[i].attached == RTE_CRYPTODEV_ATTACHED) {
 734                         int cmp;
 735
 736                         cmp = strncmp(devs[i].device->driver->name,
 737                                         driver_name,
 738                                         strlen(driver_name) + 1);
 739
 740                         if (cmp == 0)
 741                                 devices[count++] = devs[i].data->dev_id;
 742                 }
 743         }
 744
 745         return count;
 746 }
 747
 748 void *
 749 rte_cryptodev_get_sec_ctx(uint8_t dev_id)
 750 {
 751         if (dev_id < RTE_CRYPTO_MAX_DEVS &&
 752                         (rte_crypto_devices[dev_id].feature_flags &
 753                         RTE_CRYPTODEV_FF_SECURITY))
 754                 return rte_crypto_devices[dev_id].security_ctx;
 755
 756         return NULL;
 757 }
 758
 759 int
 760 rte_cryptodev_socket_id(uint8_t dev_id)
 761 {
 762         struct rte_cryptodev *dev;
 763
 764         if (!rte_cryptodev_is_valid_dev(dev_id))
 765                 return -1;
 766
 767         dev = rte_cryptodev_pmd_get_dev(dev_id);
 768
 769         return dev->data->socket_id;
 770 }
 771
 772 static inline int
 773 rte_cryptodev_data_alloc(uint8_t dev_id, struct rte_cryptodev_data **data,
 774                 int socket_id)
 775 {
 776         char mz_name[RTE_MEMZONE_NAMESIZE];
 777         const struct rte_memzone *mz;
 778         int n;
 779
 780         /* generate memzone name */
 781         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 782         if (n >= (int)sizeof(mz_name))
 783                 return -EINVAL;
 784
 785         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 786                 mz = rte_memzone_reserve(mz_name,
 787                                 sizeof(struct rte_cryptodev_data),
 788                                 socket_id, 0);
 789                 CDEV_LOG_DEBUG("PRIMARY:reserved memzone for %s (%p)",
 790                                 mz_name, mz);
 791         } else {
 792                 mz = rte_memzone_lookup(mz_name);
 793                 CDEV_LOG_DEBUG("SECONDARY:looked up memzone for %s (%p)",
 794                                 mz_name, mz);
 795         }
 796
 797         if (mz == NULL)
 798                 return -ENOMEM;
 799
 800         *data = mz->addr;
 801         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 802                 memset(*data, 0, sizeof(struct rte_cryptodev_data));
 803
 804         return 0;
 805 }
 806
 807 static inline int
 808 rte_cryptodev_data_free(uint8_t dev_id, struct rte_cryptodev_data **data)
 809 {
 810         char mz_name[RTE_MEMZONE_NAMESIZE];
 811         const struct rte_memzone *mz;
 812         int n;
 813
 814         /* generate memzone name */
 815         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 816         if (n >= (int)sizeof(mz_name))
 817                 return -EINVAL;
 818
 819         mz = rte_memzone_lookup(mz_name);
 820         if (mz == NULL)
 821                 return -ENOMEM;
 822
 823         RTE_ASSERT(*data == mz->addr);
 824         *data = NULL;
 825
 826         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 827                 CDEV_LOG_DEBUG("PRIMARY:free memzone of %s (%p)",
 828                                 mz_name, mz);
 829                 return rte_memzone_free(mz);
 830         } else {
 831                 CDEV_LOG_DEBUG("SECONDARY:don't free memzone of %s (%p)",
 832                                 mz_name, mz);
 833         }
 834
 835         return 0;
 836 }
 837
 838 static uint8_t
 839 rte_cryptodev_find_free_device_index(void)
 840 {
 841         uint8_t dev_id;
 842
 843         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++) {
 844                 if (rte_crypto_devices[dev_id].attached ==
 845                                 RTE_CRYPTODEV_DETACHED)
 846                         return dev_id;
 847         }
 848         return RTE_CRYPTO_MAX_DEVS;
 849 }
 850
 851 struct rte_cryptodev *
 852 rte_cryptodev_pmd_allocate(const char *name, int socket_id)
 853 {
 854         struct rte_cryptodev *cryptodev;
 855         uint8_t dev_id;
 856
 857         if (rte_cryptodev_pmd_get_named_dev(name) != NULL) {
 858                 CDEV_LOG_ERR("Crypto device with name %s already "
 859                                 "allocated!", name);
 860                 return NULL;
 861         }
 862
 863         dev_id = rte_cryptodev_find_free_device_index();
 864         if (dev_id == RTE_CRYPTO_MAX_DEVS) {
 865                 CDEV_LOG_ERR("Reached maximum number of crypto devices");
 866                 return NULL;
 867         }
 868
 869         cryptodev = rte_cryptodev_pmd_get_dev(dev_id);
 870
 871         if (cryptodev->data == NULL) {
 872                 struct rte_cryptodev_data **cryptodev_data =
 873                                 &cryptodev_globals.data[dev_id];
 874
 875                 int retval = rte_cryptodev_data_alloc(dev_id, cryptodev_data,
 876                                 socket_id);
 877
 878                 if (retval < 0 || *cryptodev_data == NULL)
 879                         return NULL;
 880
 881                 cryptodev->data = *cryptodev_data;
 882
 883                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 884                         strlcpy(cryptodev->data->name, name,
 885                                 RTE_CRYPTODEV_NAME_MAX_LEN);
 886
 887                         cryptodev->data->dev_id = dev_id;
 888                         cryptodev->data->socket_id = socket_id;
 889                         cryptodev->data->dev_started = 0;
 890                         CDEV_LOG_DEBUG("PRIMARY:init data");
 891                 }
 892
 893                 CDEV_LOG_DEBUG("Data for %s: dev_id %d, socket %d, started %d",
 894                                 cryptodev->data->name,
 895                                 cryptodev->data->dev_id,
 896                                 cryptodev->data->socket_id,
 897                                 cryptodev->data->dev_started);
 898
 899                 /* init user callbacks */
 900                 TAILQ_INIT(&(cryptodev->link_intr_cbs));
 901
 902                 cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 903
 904                 cryptodev_globals.nb_devs++;
 905         }
 906
 907         return cryptodev;
 908 }
 909
 910 int
 911 rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 912 {
 913         int ret;
 914         uint8_t dev_id;
 915
 916         if (cryptodev == NULL)
 917                 return -EINVAL;
 918
 919         dev_id = cryptodev->data->dev_id;
 920
 921         /* Close device only if device operations have been set */
 922         if (cryptodev->dev_ops) {
 923                 ret = rte_cryptodev_close(dev_id);
 924                 if (ret < 0)
 925                         return ret;
 926         }
 927
 928         ret = rte_cryptodev_data_free(dev_id, &cryptodev_globals.data[dev_id]);
 929         if (ret < 0)
 930                 return ret;
 931
 932         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 933         cryptodev_globals.nb_devs--;
 934         return 0;
 935 }
 936
 937 uint16_t
 938 rte_cryptodev_queue_pair_count(uint8_t dev_id)
 939 {
 940         struct rte_cryptodev *dev;
 941
 942         if (!rte_cryptodev_is_valid_device_data(dev_id)) {
 943                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 944                 return 0;
 945         }
 946
 947         dev = &rte_crypto_devices[dev_id];
 948         return dev->data->nb_queue_pairs;
 949 }
 950
 951 static int
 952 rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 953                 int socket_id)
 954 {
 955         struct rte_cryptodev_info dev_info;
 956         void **qp;
 957         unsigned i;
 958
 959         if ((dev == NULL) || (nb_qpairs < 1)) {
 960                 CDEV_LOG_ERR("invalid param: dev %p, nb_queues %u",
 961                                                         dev, nb_qpairs);
 962                 return -EINVAL;
 963         }
 964
 965         CDEV_LOG_DEBUG("Setup %d queues pairs on device %u",
 966                         nb_qpairs, dev->data->dev_id);
 967
 968         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 969
 970         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 971         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 972
 973         if (nb_qpairs > (dev_info.max_nb_queue_pairs)) {
 974                 CDEV_LOG_ERR("Invalid num queue_pairs (%u) for dev %u",
 975                                 nb_qpairs, dev->data->dev_id);
 976             return -EINVAL;
 977         }
 978
 979         if (dev->data->queue_pairs == NULL) { /* first time configuration */
 980                 dev->data->queue_pairs = rte_zmalloc_socket(
 981                                 "cryptodev->queue_pairs",
 982                                 sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
 983                                 RTE_CACHE_LINE_SIZE, socket_id);
 984
 985                 if (dev->data->queue_pairs == NULL) {
 986                         dev->data->nb_queue_pairs = 0;
 987                         CDEV_LOG_ERR("failed to get memory for qp meta data, "
 988                                                         "nb_queues %u",
 989                                                         nb_qpairs);
 990                         return -(ENOMEM);
 991                 }
 992         } else { /* re-configure */
 993                 int ret;
 994                 uint16_t old_nb_queues = dev->data->nb_queue_pairs;
 995
 996                 qp = dev->data->queue_pairs;
 997
 998                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
 999                                 -ENOTSUP);
1000
1001                 for (i = nb_qpairs; i < old_nb_queues; i++) {
1002                         ret = (*dev->dev_ops->queue_pair_release)(dev, i);
1003                         if (ret < 0)
1004                                 return ret;
1005                 }
1006
1007                 qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
1008                                 RTE_CACHE_LINE_SIZE);
1009                 if (qp == NULL) {
1010                         CDEV_LOG_ERR("failed to realloc qp meta data,"
1011                                                 " nb_queues %u", nb_qpairs);
1012                         return -(ENOMEM);
1013                 }
1014
1015                 if (nb_qpairs > old_nb_queues) {
1016                         uint16_t new_qs = nb_qpairs - old_nb_queues;
1017
1018                         memset(qp + old_nb_queues, 0,
1019                                 sizeof(qp[0]) * new_qs);
1020                 }
1021
1022                 dev->data->queue_pairs = qp;
1023
1024         }
1025         dev->data->nb_queue_pairs = nb_qpairs;
1026         return 0;
1027 }
1028
1029 int
1030 rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
1031 {
1032         struct rte_cryptodev *dev;
1033         int diag;
1034
1035         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1036                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1037                 return -EINVAL;
1038         }
1039
1040         dev = &rte_crypto_devices[dev_id];
1041
1042         if (dev->data->dev_started) {
1043                 CDEV_LOG_ERR(
1044                     "device %d must be stopped to allow configuration", dev_id);
1045                 return -EBUSY;
1046         }
1047
1048         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1049
1050         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1051         cryptodev_cb_cleanup(dev);
1052         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1053
1054         /* Setup new number of queue pairs and reconfigure device. */
1055         diag = rte_cryptodev_queue_pairs_config(dev, config->nb_queue_pairs,
1056                         config->socket_id);
1057         if (diag != 0) {
1058                 CDEV_LOG_ERR("dev%d rte_crypto_dev_queue_pairs_config = %d",
1059                                 dev_id, diag);
1060                 return diag;
1061         }
1062
1063         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1064         diag = cryptodev_cb_init(dev);
1065         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1066         if (diag) {
1067                 CDEV_LOG_ERR("Callback init failed for dev_id=%d", dev_id);
1068                 return diag;
1069         }
1070
1071         rte_cryptodev_trace_configure(dev_id, config);
1072         return (*dev->dev_ops->dev_configure)(dev, config);
1073 }
1074
1075 int
1076 rte_cryptodev_start(uint8_t dev_id)
1077 {
1078         struct rte_cryptodev *dev;
1079         int diag;
1080
1081         CDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id);
1082
1083         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1084                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1085                 return -EINVAL;
1086         }
1087
1088         dev = &rte_crypto_devices[dev_id];
1089
1090         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1091
1092         if (dev->data->dev_started != 0) {
1093                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already started",
1094                         dev_id);
1095                 return 0;
1096         }
1097
1098         diag = (*dev->dev_ops->dev_start)(dev);
1099         rte_cryptodev_trace_start(dev_id, diag);
1100         if (diag == 0)
1101                 dev->data->dev_started = 1;
1102         else
1103                 return diag;
1104
1105         return 0;
1106 }
1107
1108 void
1109 rte_cryptodev_stop(uint8_t dev_id)
1110 {
1111         struct rte_cryptodev *dev;
1112
1113         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1114                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1115                 return;
1116         }
1117
1118         dev = &rte_crypto_devices[dev_id];
1119
1120         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1121
1122         if (dev->data->dev_started == 0) {
1123                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already stopped",
1124                         dev_id);
1125                 return;
1126         }
1127
1128         (*dev->dev_ops->dev_stop)(dev);
1129         rte_cryptodev_trace_stop(dev_id);
1130         dev->data->dev_started = 0;
1131 }
1132
1133 int
1134 rte_cryptodev_close(uint8_t dev_id)
1135 {
1136         struct rte_cryptodev *dev;
1137         int retval;
1138
1139         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1140                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1141                 return -1;
1142         }
1143
1144         dev = &rte_crypto_devices[dev_id];
1145
1146         /* Device must be stopped before it can be closed */
1147         if (dev->data->dev_started == 1) {
1148                 CDEV_LOG_ERR("Device %u must be stopped before closing",
1149                                 dev_id);
1150                 return -EBUSY;
1151         }
1152
1153         /* We can't close the device if there are outstanding sessions in use */
1154         if (dev->data->session_pool != NULL) {
1155                 if (!rte_mempool_full(dev->data->session_pool)) {
1156                         CDEV_LOG_ERR("dev_id=%u close failed, session mempool "
1157                                         "has sessions still in use, free "
1158                                         "all sessions before calling close",
1159                                         (unsigned)dev_id);
1160                         return -EBUSY;
1161                 }
1162         }
1163
1164         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
1165         retval = (*dev->dev_ops->dev_close)(dev);
1166         rte_cryptodev_trace_close(dev_id, retval);
1167
1168         if (retval < 0)
1169                 return retval;
1170
1171         return 0;
1172 }
1173
1174 int
1175 rte_cryptodev_get_qp_status(uint8_t dev_id, uint16_t queue_pair_id)
1176 {
1177         struct rte_cryptodev *dev;
1178
1179         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1180                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1181                 return -EINVAL;
1182         }
1183
1184         dev = &rte_crypto_devices[dev_id];
1185         if (queue_pair_id >= dev->data->nb_queue_pairs) {
1186                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
1187                 return -EINVAL;
1188         }
1189         void **qps = dev->data->queue_pairs;
1190
1191         if (qps[queue_pair_id]) {
1192                 CDEV_LOG_DEBUG("qp %d on dev %d is initialised",
1193                         queue_pair_id, dev_id);
1194                 return 1;
1195         }
1196
1197         CDEV_LOG_DEBUG("qp %d on dev %d is not initialised",
1198                 queue_pair_id, dev_id);
1199
1200         return 0;
1201 }
1202
1203 int
1204 rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
1205                 const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
1206
1207 {
1208         struct rte_cryptodev *dev;
1209
1210         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1211                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1212                 return -EINVAL;
1213         }
1214
1215         dev = &rte_crypto_devices[dev_id];
1216         if (queue_pair_id >= dev->data->nb_queue_pairs) {
1217                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
1218                 return -EINVAL;
1219         }
1220
1221         if (!qp_conf) {
1222                 CDEV_LOG_ERR("qp_conf cannot be NULL\n");
1223                 return -EINVAL;
1224         }
1225
1226         if ((qp_conf->mp_session && !qp_conf->mp_session_private) ||
1227                         (!qp_conf->mp_session && qp_conf->mp_session_private)) {
1228                 CDEV_LOG_ERR("Invalid mempools\n");
1229                 return -EINVAL;
1230         }
1231
1232         if (qp_conf->mp_session) {
1233                 struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1234                 uint32_t obj_size = qp_conf->mp_session->elt_size;
1235                 uint32_t obj_priv_size = qp_conf->mp_session_private->elt_size;
1236                 struct rte_cryptodev_sym_session s = {0};
1237
1238                 pool_priv = rte_mempool_get_priv(qp_conf->mp_session);
1239                 if (!pool_priv || qp_conf->mp_session->private_data_size <
1240                                 sizeof(*pool_priv)) {
1241                         CDEV_LOG_ERR("Invalid mempool\n");
1242                         return -EINVAL;
1243                 }
1244
1245                 s.nb_drivers = pool_priv->nb_drivers;
1246                 s.user_data_sz = pool_priv->user_data_sz;
1247
1248                 if ((rte_cryptodev_sym_get_existing_header_session_size(&s) >
1249                         obj_size) || (s.nb_drivers <= dev->driver_id) ||
1250                         rte_cryptodev_sym_get_private_session_size(dev_id) >
1251                                 obj_priv_size) {
1252                         CDEV_LOG_ERR("Invalid mempool\n");
1253                         return -EINVAL;
1254                 }
1255         }
1256
1257         if (dev->data->dev_started) {
1258                 CDEV_LOG_ERR(
1259                     "device %d must be stopped to allow configuration", dev_id);
1260                 return -EBUSY;
1261         }
1262
1263         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
1264
1265         rte_cryptodev_trace_queue_pair_setup(dev_id, queue_pair_id, qp_conf);
1266         return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id, qp_conf,
1267                         socket_id);
1268 }
1269
1270 struct rte_cryptodev_cb *
1271 rte_cryptodev_add_enq_callback(uint8_t dev_id,
1272                                uint16_t qp_id,
1273                                rte_cryptodev_callback_fn cb_fn,
1274                                void *cb_arg)
1275 {
1276         struct rte_cryptodev *dev;
1277         struct rte_cryptodev_cb_rcu *list;
1278         struct rte_cryptodev_cb *cb, *tail;
1279
1280         if (!cb_fn) {
1281                 CDEV_LOG_ERR("Callback is NULL on dev_id=%d", dev_id);
1282                 rte_errno = EINVAL;
1283                 return NULL;
1284         }
1285
1286         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1287                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1288                 rte_errno = ENODEV;
1289                 return NULL;
1290         }
1291
1292         dev = &rte_crypto_devices[dev_id];
1293         if (qp_id >= dev->data->nb_queue_pairs) {
1294                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1295                 rte_errno = ENODEV;
1296                 return NULL;
1297         }
1298
1299         cb = rte_zmalloc(NULL, sizeof(*cb), 0);
1300         if (cb == NULL) {
1301                 CDEV_LOG_ERR("Failed to allocate memory for callback on "
1302                              "dev=%d, queue_pair_id=%d", dev_id, qp_id);
1303                 rte_errno = ENOMEM;
1304                 return NULL;
1305         }
1306
1307         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1308
1309         cb->fn = cb_fn;
1310         cb->arg = cb_arg;
1311
1312         /* Add the callbacks in fifo order. */
1313         list = &dev->enq_cbs[qp_id];
1314         tail = list->next;
1315
1316         if (tail) {
1317                 while (tail->next)
1318                         tail = tail->next;
1319                 /* Stores to cb->fn and cb->param should complete before
1320                  * cb is visible to data plane.
1321                  */
1322                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
1323         } else {
1324                 /* Stores to cb->fn and cb->param should complete before
1325                  * cb is visible to data plane.
1326                  */
1327                 __atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
1328         }
1329
1330         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1331
1332         return cb;
1333 }
1334
1335 int
1336 rte_cryptodev_remove_enq_callback(uint8_t dev_id,
1337                                   uint16_t qp_id,
1338                                   struct rte_cryptodev_cb *cb)
1339 {
1340         struct rte_cryptodev *dev;
1341         struct rte_cryptodev_cb **prev_cb, *curr_cb;
1342         struct rte_cryptodev_cb_rcu *list;
1343         int ret;
1344
1345         ret = -EINVAL;
1346
1347         if (!cb) {
1348                 CDEV_LOG_ERR("Callback is NULL");
1349                 return -EINVAL;
1350         }
1351
1352         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1353                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1354                 return -ENODEV;
1355         }
1356
1357         dev = &rte_crypto_devices[dev_id];
1358         if (qp_id >= dev->data->nb_queue_pairs) {
1359                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1360                 return -ENODEV;
1361         }
1362
1363         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1364         if (dev->enq_cbs == NULL) {
1365                 CDEV_LOG_ERR("Callback not initialized");
1366                 goto cb_err;
1367         }
1368
1369         list = &dev->enq_cbs[qp_id];
1370         if (list == NULL) {
1371                 CDEV_LOG_ERR("Callback list is NULL");
1372                 goto cb_err;
1373         }
1374
1375         if (list->qsbr == NULL) {
1376                 CDEV_LOG_ERR("Rcu qsbr is NULL");
1377                 goto cb_err;
1378         }
1379
1380         prev_cb = &list->next;
1381         for (; *prev_cb != NULL; prev_cb = &curr_cb->next) {
1382                 curr_cb = *prev_cb;
1383                 if (curr_cb == cb) {
1384                         /* Remove the user cb from the callback list. */
1385                         __atomic_store_n(prev_cb, curr_cb->next,
1386                                 __ATOMIC_RELAXED);
1387                         ret = 0;
1388                         break;
1389                 }
1390         }
1391
1392         if (!ret) {
1393                 /* Call sync with invalid thread id as this is part of
1394                  * control plane API
1395                  */
1396                 rte_rcu_qsbr_synchronize(list->qsbr, RTE_QSBR_THRID_INVALID);
1397                 rte_free(cb);
1398         }
1399
1400 cb_err:
1401         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1402         return ret;
1403 }
1404
1405 struct rte_cryptodev_cb *
1406 rte_cryptodev_add_deq_callback(uint8_t dev_id,
1407                                uint16_t qp_id,
1408                                rte_cryptodev_callback_fn cb_fn,
1409                                void *cb_arg)
1410 {
1411         struct rte_cryptodev *dev;
1412         struct rte_cryptodev_cb_rcu *list;
1413         struct rte_cryptodev_cb *cb, *tail;
1414
1415         if (!cb_fn) {
1416                 CDEV_LOG_ERR("Callback is NULL on dev_id=%d", dev_id);
1417                 rte_errno = EINVAL;
1418                 return NULL;
1419         }
1420
1421         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1422                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1423                 rte_errno = ENODEV;
1424                 return NULL;
1425         }
1426
1427         dev = &rte_crypto_devices[dev_id];
1428         if (qp_id >= dev->data->nb_queue_pairs) {
1429                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1430                 rte_errno = ENODEV;
1431                 return NULL;
1432         }
1433
1434         cb = rte_zmalloc(NULL, sizeof(*cb), 0);
1435         if (cb == NULL) {
1436                 CDEV_LOG_ERR("Failed to allocate memory for callback on "
1437                              "dev=%d, queue_pair_id=%d", dev_id, qp_id);
1438                 rte_errno = ENOMEM;
1439                 return NULL;
1440         }
1441
1442         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1443
1444         cb->fn = cb_fn;
1445         cb->arg = cb_arg;
1446
1447         /* Add the callbacks in fifo order. */
1448         list = &dev->deq_cbs[qp_id];
1449         tail = list->next;
1450
1451         if (tail) {
1452                 while (tail->next)
1453                         tail = tail->next;
1454                 /* Stores to cb->fn and cb->param should complete before
1455                  * cb is visible to data plane.
1456                  */
1457                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
1458         } else {
1459                 /* Stores to cb->fn and cb->param should complete before
1460                  * cb is visible to data plane.
1461                  */
1462                 __atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
1463         }
1464
1465         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1466
1467         return cb;
1468 }
1469
1470 int
1471 rte_cryptodev_remove_deq_callback(uint8_t dev_id,
1472                                   uint16_t qp_id,
1473                                   struct rte_cryptodev_cb *cb)
1474 {
1475         struct rte_cryptodev *dev;
1476         struct rte_cryptodev_cb **prev_cb, *curr_cb;
1477         struct rte_cryptodev_cb_rcu *list;
1478         int ret;
1479
1480         ret = -EINVAL;
1481
1482         if (!cb) {
1483                 CDEV_LOG_ERR("Callback is NULL");
1484                 return -EINVAL;
1485         }
1486
1487         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1488                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1489                 return -ENODEV;
1490         }
1491
1492         dev = &rte_crypto_devices[dev_id];
1493         if (qp_id >= dev->data->nb_queue_pairs) {
1494                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1495                 return -ENODEV;
1496         }
1497
1498         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1499         if (dev->enq_cbs == NULL) {
1500                 CDEV_LOG_ERR("Callback not initialized");
1501                 goto cb_err;
1502         }
1503
1504         list = &dev->deq_cbs[qp_id];
1505         if (list == NULL) {
1506                 CDEV_LOG_ERR("Callback list is NULL");
1507                 goto cb_err;
1508         }
1509
1510         if (list->qsbr == NULL) {
1511                 CDEV_LOG_ERR("Rcu qsbr is NULL");
1512                 goto cb_err;
1513         }
1514
1515         prev_cb = &list->next;
1516         for (; *prev_cb != NULL; prev_cb = &curr_cb->next) {
1517                 curr_cb = *prev_cb;
1518                 if (curr_cb == cb) {
1519                         /* Remove the user cb from the callback list. */
1520                         __atomic_store_n(prev_cb, curr_cb->next,
1521                                 __ATOMIC_RELAXED);
1522                         ret = 0;
1523                         break;
1524                 }
1525         }
1526
1527         if (!ret) {
1528                 /* Call sync with invalid thread id as this is part of
1529                  * control plane API
1530                  */
1531                 rte_rcu_qsbr_synchronize(list->qsbr, RTE_QSBR_THRID_INVALID);
1532                 rte_free(cb);
1533         }
1534
1535 cb_err:
1536         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1537         return ret;
1538 }
1539
1540 int
1541 rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats)
1542 {
1543         struct rte_cryptodev *dev;
1544
1545         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1546                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1547                 return -ENODEV;
1548         }
1549
1550         if (stats == NULL) {
1551                 CDEV_LOG_ERR("Invalid stats ptr");
1552                 return -EINVAL;
1553         }
1554
1555         dev = &rte_crypto_devices[dev_id];
1556         memset(stats, 0, sizeof(*stats));
1557
1558         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1559         (*dev->dev_ops->stats_get)(dev, stats);
1560         return 0;
1561 }
1562
1563 void
1564 rte_cryptodev_stats_reset(uint8_t dev_id)
1565 {
1566         struct rte_cryptodev *dev;
1567
1568         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1569                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1570                 return;
1571         }
1572
1573         dev = &rte_crypto_devices[dev_id];
1574
1575         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
1576         (*dev->dev_ops->stats_reset)(dev);
1577 }
1578
1579 void
1580 rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
1581 {
1582         struct rte_cryptodev *dev;
1583
1584         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1585                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1586                 return;
1587         }
1588
1589         dev = &rte_crypto_devices[dev_id];
1590
1591         memset(dev_info, 0, sizeof(struct rte_cryptodev_info));
1592
1593         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
1594         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
1595
1596         dev_info->driver_name = dev->device->driver->name;
1597         dev_info->device = dev->device;
1598 }
1599
1600 int
1601 rte_cryptodev_callback_register(uint8_t dev_id,
1602                         enum rte_cryptodev_event_type event,
1603                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
1604 {
1605         struct rte_cryptodev *dev;
1606         struct rte_cryptodev_callback *user_cb;
1607
1608         if (!cb_fn)
1609                 return -EINVAL;
1610
1611         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1612                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1613                 return -EINVAL;
1614         }
1615
1616         dev = &rte_crypto_devices[dev_id];
1617         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1618
1619         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
1620                 if (user_cb->cb_fn == cb_fn &&
1621                         user_cb->cb_arg == cb_arg &&
1622                         user_cb->event == event) {
1623                         break;
1624                 }
1625         }
1626
1627         /* create a new callback. */
1628         if (user_cb == NULL) {
1629                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
1630                                 sizeof(struct rte_cryptodev_callback), 0);
1631                 if (user_cb != NULL) {
1632                         user_cb->cb_fn = cb_fn;
1633                         user_cb->cb_arg = cb_arg;
1634                         user_cb->event = event;
1635                         TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
1636                 }
1637         }
1638
1639         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1640         return (user_cb == NULL) ? -ENOMEM : 0;
1641 }
1642
1643 int
1644 rte_cryptodev_callback_unregister(uint8_t dev_id,
1645                         enum rte_cryptodev_event_type event,
1646                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
1647 {
1648         int ret;
1649         struct rte_cryptodev *dev;
1650         struct rte_cryptodev_callback *cb, *next;
1651
1652         if (!cb_fn)
1653                 return -EINVAL;
1654
1655         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1656                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1657                 return -EINVAL;
1658         }
1659
1660         dev = &rte_crypto_devices[dev_id];
1661         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1662
1663         ret = 0;
1664         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
1665
1666                 next = TAILQ_NEXT(cb, next);
1667
1668                 if (cb->cb_fn != cb_fn || cb->event != event ||
1669                                 (cb->cb_arg != (void *)-1 &&
1670                                 cb->cb_arg != cb_arg))
1671                         continue;
1672
1673                 /*
1674                  * if this callback is not executing right now,
1675                  * then remove it.
1676                  */
1677                 if (cb->active == 0) {
1678                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
1679                         rte_free(cb);
1680                 } else {
1681                         ret = -EAGAIN;
1682                 }
1683         }
1684
1685         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1686         return ret;
1687 }
1688
1689 void
1690 rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev,
1691         enum rte_cryptodev_event_type event)
1692 {
1693         struct rte_cryptodev_callback *cb_lst;
1694         struct rte_cryptodev_callback dev_cb;
1695
1696         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1697         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
1698                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1699                         continue;
1700                 dev_cb = *cb_lst;
1701                 cb_lst->active = 1;
1702                 rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1703                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1704                                                 dev_cb.cb_arg);
1705                 rte_spinlock_lock(&rte_cryptodev_cb_lock);
1706                 cb_lst->active = 0;
1707         }
1708         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1709 }
1710
1711 int
1712 rte_cryptodev_sym_session_init(uint8_t dev_id,
1713                 struct rte_cryptodev_sym_session *sess,
1714                 struct rte_crypto_sym_xform *xforms,
1715                 struct rte_mempool *mp)
1716 {
1717         struct rte_cryptodev *dev;
1718         uint32_t sess_priv_sz = rte_cryptodev_sym_get_private_session_size(
1719                         dev_id);
1720         uint8_t index;
1721         int ret;
1722
1723         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1724                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1725                 return -EINVAL;
1726         }
1727
1728         dev = rte_cryptodev_pmd_get_dev(dev_id);
1729
1730         if (sess == NULL || xforms == NULL || dev == NULL || mp == NULL)
1731                 return -EINVAL;
1732
1733         if (mp->elt_size < sess_priv_sz)
1734                 return -EINVAL;
1735
1736         index = dev->driver_id;
1737         if (index >= sess->nb_drivers)
1738                 return -EINVAL;
1739
1740         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_configure, -ENOTSUP);
1741
1742         if (sess->sess_data[index].refcnt == 0) {
1743                 ret = dev->dev_ops->sym_session_configure(dev, xforms,
1744                                                         sess, mp);
1745                 if (ret < 0) {
1746                         CDEV_LOG_ERR(
1747                                 "dev_id %d failed to configure session details",
1748                                 dev_id);
1749                         return ret;
1750                 }
1751         }
1752
1753         rte_cryptodev_trace_sym_session_init(dev_id, sess, xforms, mp);
1754         sess->sess_data[index].refcnt++;
1755         return 0;
1756 }
1757
1758 int
1759 rte_cryptodev_asym_session_init(uint8_t dev_id,
1760                 struct rte_cryptodev_asym_session *sess,
1761                 struct rte_crypto_asym_xform *xforms,
1762                 struct rte_mempool *mp)
1763 {
1764         struct rte_cryptodev *dev;
1765         uint8_t index;
1766         int ret;
1767
1768         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1769                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1770                 return -EINVAL;
1771         }
1772
1773         dev = rte_cryptodev_pmd_get_dev(dev_id);
1774
1775         if (sess == NULL || xforms == NULL || dev == NULL)
1776                 return -EINVAL;
1777
1778         index = dev->driver_id;
1779
1780         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_configure,
1781                                 -ENOTSUP);
1782
1783         if (sess->sess_private_data[index] == NULL) {
1784                 ret = dev->dev_ops->asym_session_configure(dev,
1785                                                         xforms,
1786                                                         sess, mp);
1787                 if (ret < 0) {
1788                         CDEV_LOG_ERR(
1789                                 "dev_id %d failed to configure session details",
1790                                 dev_id);
1791                         return ret;
1792                 }
1793         }
1794
1795         rte_cryptodev_trace_asym_session_init(dev_id, sess, xforms, mp);
1796         return 0;
1797 }
1798
1799 struct rte_mempool *
1800 rte_cryptodev_sym_session_pool_create(const char *name, uint32_t nb_elts,
1801         uint32_t elt_size, uint32_t cache_size, uint16_t user_data_size,
1802         int socket_id)
1803 {
1804         struct rte_mempool *mp;
1805         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1806         uint32_t obj_sz;
1807
1808         obj_sz = rte_cryptodev_sym_get_header_session_size() + user_data_size;
1809         if (obj_sz > elt_size)
1810                 CDEV_LOG_INFO("elt_size %u is expanded to %u\n", elt_size,
1811                                 obj_sz);
1812         else
1813                 obj_sz = elt_size;
1814
1815         mp = rte_mempool_create(name, nb_elts, obj_sz, cache_size,
1816                         (uint32_t)(sizeof(*pool_priv)),
1817                         NULL, NULL, NULL, NULL,
1818                         socket_id, 0);
1819         if (mp == NULL) {
1820                 CDEV_LOG_ERR("%s(name=%s) failed, rte_errno=%d\n",
1821                         __func__, name, rte_errno);
1822                 return NULL;
1823         }
1824
1825         pool_priv = rte_mempool_get_priv(mp);
1826         if (!pool_priv) {
1827                 CDEV_LOG_ERR("%s(name=%s) failed to get private data\n",
1828                         __func__, name);
1829                 rte_mempool_free(mp);
1830                 return NULL;
1831         }
1832
1833         pool_priv->nb_drivers = nb_drivers;
1834         pool_priv->user_data_sz = user_data_size;
1835
1836         rte_cryptodev_trace_sym_session_pool_create(name, nb_elts,
1837                 elt_size, cache_size, user_data_size, mp);
1838         return mp;
1839 }
1840
1841 static unsigned int
1842 rte_cryptodev_sym_session_data_size(struct rte_cryptodev_sym_session *sess)
1843 {
1844         return (sizeof(sess->sess_data[0]) * sess->nb_drivers) +
1845                         sess->user_data_sz;
1846 }
1847
1848 static uint8_t
1849 rte_cryptodev_sym_is_valid_session_pool(struct rte_mempool *mp)
1850 {
1851         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1852
1853         if (!mp)
1854                 return 0;
1855
1856         pool_priv = rte_mempool_get_priv(mp);
1857
1858         if (!pool_priv || mp->private_data_size < sizeof(*pool_priv) ||
1859                         pool_priv->nb_drivers != nb_drivers ||
1860                         mp->elt_size <
1861                                 rte_cryptodev_sym_get_header_session_size()
1862                                 + pool_priv->user_data_sz)
1863                 return 0;
1864
1865         return 1;
1866 }
1867
1868 struct rte_cryptodev_sym_session *
1869 rte_cryptodev_sym_session_create(struct rte_mempool *mp)
1870 {
1871         struct rte_cryptodev_sym_session *sess;
1872         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1873
1874         if (!rte_cryptodev_sym_is_valid_session_pool(mp)) {
1875                 CDEV_LOG_ERR("Invalid mempool\n");
1876                 return NULL;
1877         }
1878
1879         pool_priv = rte_mempool_get_priv(mp);
1880
1881         /* Allocate a session structure from the session pool */
1882         if (rte_mempool_get(mp, (void **)&sess)) {
1883                 CDEV_LOG_ERR("couldn't get object from session mempool");
1884                 return NULL;
1885         }
1886
1887         sess->nb_drivers = pool_priv->nb_drivers;
1888         sess->user_data_sz = pool_priv->user_data_sz;
1889         sess->opaque_data = 0;
1890
1891         /* Clear device session pointer.
1892          * Include the flag indicating presence of user data
1893          */
1894         memset(sess->sess_data, 0,
1895                         rte_cryptodev_sym_session_data_size(sess));
1896
1897         rte_cryptodev_trace_sym_session_create(mp, sess);
1898         return sess;
1899 }
1900
1901 struct rte_cryptodev_asym_session *
1902 rte_cryptodev_asym_session_create(struct rte_mempool *mp)
1903 {
1904         struct rte_cryptodev_asym_session *sess;
1905         unsigned int session_size =
1906                         rte_cryptodev_asym_get_header_session_size();
1907
1908         if (!mp) {
1909                 CDEV_LOG_ERR("invalid mempool\n");
1910                 return NULL;
1911         }
1912
1913         /* Verify if provided mempool can hold elements big enough. */
1914         if (mp->elt_size < session_size) {
1915                 CDEV_LOG_ERR(
1916                         "mempool elements too small to hold session objects");
1917                 return NULL;
1918         }
1919
1920         /* Allocate a session structure from the session pool */
1921         if (rte_mempool_get(mp, (void **)&sess)) {
1922                 CDEV_LOG_ERR("couldn't get object from session mempool");
1923                 return NULL;
1924         }
1925
1926         /* Clear device session pointer.
1927          * Include the flag indicating presence of private data
1928          */
1929         memset(sess, 0, session_size);
1930
1931         rte_cryptodev_trace_asym_session_create(mp, sess);
1932         return sess;
1933 }
1934
1935 int
1936 rte_cryptodev_sym_session_clear(uint8_t dev_id,
1937                 struct rte_cryptodev_sym_session *sess)
1938 {
1939         struct rte_cryptodev *dev;
1940         uint8_t driver_id;
1941
1942         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1943                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1944                 return -EINVAL;
1945         }
1946
1947         dev = rte_cryptodev_pmd_get_dev(dev_id);
1948
1949         if (dev == NULL || sess == NULL)
1950                 return -EINVAL;
1951
1952         driver_id = dev->driver_id;
1953         if (sess->sess_data[driver_id].refcnt == 0)
1954                 return 0;
1955         if (--sess->sess_data[driver_id].refcnt != 0)
1956                 return -EBUSY;
1957
1958         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_clear, -ENOTSUP);
1959
1960         dev->dev_ops->sym_session_clear(dev, sess);
1961
1962         rte_cryptodev_trace_sym_session_clear(dev_id, sess);
1963         return 0;
1964 }
1965
1966 int
1967 rte_cryptodev_asym_session_clear(uint8_t dev_id,
1968                 struct rte_cryptodev_asym_session *sess)
1969 {
1970         struct rte_cryptodev *dev;
1971
1972         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1973                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1974                 return -EINVAL;
1975         }
1976
1977         dev = rte_cryptodev_pmd_get_dev(dev_id);
1978
1979         if (dev == NULL || sess == NULL)
1980                 return -EINVAL;
1981
1982         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_clear, -ENOTSUP);
1983
1984         dev->dev_ops->asym_session_clear(dev, sess);
1985
1986         rte_cryptodev_trace_sym_session_clear(dev_id, sess);
1987         return 0;
1988 }
1989
1990 int
1991 rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess)
1992 {
1993         uint8_t i;
1994         struct rte_mempool *sess_mp;
1995
1996         if (sess == NULL)
1997                 return -EINVAL;
1998
1999         /* Check that all device private data has been freed */
2000         for (i = 0; i < sess->nb_drivers; i++) {
2001                 if (sess->sess_data[i].refcnt != 0)
2002                         return -EBUSY;
2003         }
2004
2005         /* Return session to mempool */
2006         sess_mp = rte_mempool_from_obj(sess);
2007         rte_mempool_put(sess_mp, sess);
2008
2009         rte_cryptodev_trace_sym_session_free(sess);
2010         return 0;
2011 }
2012
2013 int
2014 rte_cryptodev_asym_session_free(struct rte_cryptodev_asym_session *sess)
2015 {
2016         uint8_t i;
2017         void *sess_priv;
2018         struct rte_mempool *sess_mp;
2019
2020         if (sess == NULL)
2021                 return -EINVAL;
2022
2023         /* Check that all device private data has been freed */
2024         for (i = 0; i < nb_drivers; i++) {
2025                 sess_priv = get_asym_session_private_data(sess, i);
2026                 if (sess_priv != NULL)
2027                         return -EBUSY;
2028         }
2029
2030         /* Return session to mempool */
2031         sess_mp = rte_mempool_from_obj(sess);
2032         rte_mempool_put(sess_mp, sess);
2033
2034         rte_cryptodev_trace_asym_session_free(sess);
2035         return 0;
2036 }
2037
2038 unsigned int
2039 rte_cryptodev_sym_get_header_session_size(void)
2040 {
2041         /*
2042          * Header contains pointers to the private data of all registered
2043          * drivers and all necessary information to ensure safely clear
2044          * or free al session.
2045          */
2046         struct rte_cryptodev_sym_session s = {0};
2047
2048         s.nb_drivers = nb_drivers;
2049
2050         return (unsigned int)(sizeof(s) +
2051                         rte_cryptodev_sym_session_data_size(&s));
2052 }
2053
2054 unsigned int
2055 rte_cryptodev_sym_get_existing_header_session_size(
2056                 struct rte_cryptodev_sym_session *sess)
2057 {
2058         if (!sess)
2059                 return 0;
2060         else
2061                 return (unsigned int)(sizeof(*sess) +
2062                                 rte_cryptodev_sym_session_data_size(sess));
2063 }
2064
2065 unsigned int
2066 rte_cryptodev_asym_get_header_session_size(void)
2067 {
2068         /*
2069          * Header contains pointers to the private data
2070          * of all registered drivers, and a flag which
2071          * indicates presence of private data
2072          */
2073         return ((sizeof(void *) * nb_drivers) + sizeof(uint8_t));
2074 }
2075
2076 unsigned int
2077 rte_cryptodev_sym_get_private_session_size(uint8_t dev_id)
2078 {
2079         struct rte_cryptodev *dev;
2080         unsigned int priv_sess_size;
2081
2082         if (!rte_cryptodev_is_valid_dev(dev_id))
2083                 return 0;
2084
2085         dev = rte_cryptodev_pmd_get_dev(dev_id);
2086
2087         if (*dev->dev_ops->sym_session_get_size == NULL)
2088                 return 0;
2089
2090         priv_sess_size = (*dev->dev_ops->sym_session_get_size)(dev);
2091
2092         return priv_sess_size;
2093 }
2094
2095 unsigned int
2096 rte_cryptodev_asym_get_private_session_size(uint8_t dev_id)
2097 {
2098         struct rte_cryptodev *dev;
2099         unsigned int header_size = sizeof(void *) * nb_drivers;
2100         unsigned int priv_sess_size;
2101
2102         if (!rte_cryptodev_is_valid_dev(dev_id))
2103                 return 0;
2104
2105         dev = rte_cryptodev_pmd_get_dev(dev_id);
2106
2107         if (*dev->dev_ops->asym_session_get_size == NULL)
2108                 return 0;
2109
2110         priv_sess_size = (*dev->dev_ops->asym_session_get_size)(dev);
2111         if (priv_sess_size < header_size)
2112                 return header_size;
2113
2114         return priv_sess_size;
2115
2116 }
2117
2118 int
2119 rte_cryptodev_sym_session_set_user_data(
2120                                         struct rte_cryptodev_sym_session *sess,
2121                                         void *data,
2122                                         uint16_t size)
2123 {
2124         if (sess == NULL)
2125                 return -EINVAL;
2126
2127         if (sess->user_data_sz < size)
2128                 return -ENOMEM;
2129
2130         rte_memcpy(sess->sess_data + sess->nb_drivers, data, size);
2131         return 0;
2132 }
2133
2134 void *
2135 rte_cryptodev_sym_session_get_user_data(
2136                                         struct rte_cryptodev_sym_session *sess)
2137 {
2138         if (sess == NULL || sess->user_data_sz == 0)
2139                 return NULL;
2140
2141         return (void *)(sess->sess_data + sess->nb_drivers);
2142 }
2143
2144 static inline void
2145 sym_crypto_fill_status(struct rte_crypto_sym_vec *vec, int32_t errnum)
2146 {
2147         uint32_t i;
2148         for (i = 0; i < vec->num; i++)
2149                 vec->status[i] = errnum;
2150 }
2151
2152 uint32_t
2153 rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
2154         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs,
2155         struct rte_crypto_sym_vec *vec)
2156 {
2157         struct rte_cryptodev *dev;
2158
2159         if (!rte_cryptodev_is_valid_dev(dev_id)) {
2160                 sym_crypto_fill_status(vec, EINVAL);
2161                 return 0;
2162         }
2163
2164         dev = rte_cryptodev_pmd_get_dev(dev_id);
2165
2166         if (*dev->dev_ops->sym_cpu_process == NULL ||
2167                 !(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO)) {
2168                 sym_crypto_fill_status(vec, ENOTSUP);
2169                 return 0;
2170         }
2171
2172         return dev->dev_ops->sym_cpu_process(dev, sess, ofs, vec);
2173 }
2174
2175 int
2176 rte_cryptodev_get_raw_dp_ctx_size(uint8_t dev_id)
2177 {
2178         struct rte_cryptodev *dev;
2179         int32_t size = sizeof(struct rte_crypto_raw_dp_ctx);
2180         int32_t priv_size;
2181
2182         if (!rte_cryptodev_is_valid_dev(dev_id))
2183                 return -EINVAL;
2184
2185         dev = rte_cryptodev_pmd_get_dev(dev_id);
2186
2187         if (*dev->dev_ops->sym_get_raw_dp_ctx_size == NULL ||
2188                 !(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)) {
2189                 return -ENOTSUP;
2190         }
2191
2192         priv_size = (*dev->dev_ops->sym_get_raw_dp_ctx_size)(dev);
2193         if (priv_size < 0)
2194                 return -ENOTSUP;
2195
2196         return RTE_ALIGN_CEIL((size + priv_size), 8);
2197 }
2198
2199 int
2200 rte_cryptodev_configure_raw_dp_ctx(uint8_t dev_id, uint16_t qp_id,
2201         struct rte_crypto_raw_dp_ctx *ctx,
2202         enum rte_crypto_op_sess_type sess_type,
2203         union rte_cryptodev_session_ctx session_ctx,
2204         uint8_t is_update)
2205 {
2206         struct rte_cryptodev *dev;
2207
2208         if (!rte_cryptodev_get_qp_status(dev_id, qp_id))
2209                 return -EINVAL;
2210
2211         dev = rte_cryptodev_pmd_get_dev(dev_id);
2212         if (!(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)
2213                         || dev->dev_ops->sym_configure_raw_dp_ctx == NULL)
2214                 return -ENOTSUP;
2215
2216         return (*dev->dev_ops->sym_configure_raw_dp_ctx)(dev, qp_id, ctx,
2217                         sess_type, session_ctx, is_update);
2218 }
2219
2220 uint32_t
2221 rte_cryptodev_raw_enqueue_burst(struct rte_crypto_raw_dp_ctx *ctx,
2222         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
2223         void **user_data, int *enqueue_status)
2224 {
2225         return (*ctx->enqueue_burst)(ctx->qp_data, ctx->drv_ctx_data, vec,
2226                         ofs, user_data, enqueue_status);
2227 }
2228
2229 int
2230 rte_cryptodev_raw_enqueue_done(struct rte_crypto_raw_dp_ctx *ctx,
2231                 uint32_t n)
2232 {
2233         return (*ctx->enqueue_done)(ctx->qp_data, ctx->drv_ctx_data, n);
2234 }
2235
2236 uint32_t
2237 rte_cryptodev_raw_dequeue_burst(struct rte_crypto_raw_dp_ctx *ctx,
2238         rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
2239         uint32_t max_nb_to_dequeue,
2240         rte_cryptodev_raw_post_dequeue_t post_dequeue,
2241         void **out_user_data, uint8_t is_user_data_array,
2242         uint32_t *n_success_jobs, int *status)
2243 {
2244         return (*ctx->dequeue_burst)(ctx->qp_data, ctx->drv_ctx_data,
2245                 get_dequeue_count, max_nb_to_dequeue, post_dequeue,
2246                 out_user_data, is_user_data_array, n_success_jobs, status);
2247 }
2248
2249 int
2250 rte_cryptodev_raw_dequeue_done(struct rte_crypto_raw_dp_ctx *ctx,
2251                 uint32_t n)
2252 {
2253         return (*ctx->dequeue_done)(ctx->qp_data, ctx->drv_ctx_data, n);
2254 }
2255
2256 /** Initialise rte_crypto_op mempool element */
2257 static void
2258 rte_crypto_op_init(struct rte_mempool *mempool,
2259                 void *opaque_arg,
2260                 void *_op_data,
2261                 __rte_unused unsigned i)
2262 {
2263         struct rte_crypto_op *op = _op_data;
2264         enum rte_crypto_op_type type = *(enum rte_crypto_op_type *)opaque_arg;
2265
2266         memset(_op_data, 0, mempool->elt_size);
2267
2268         __rte_crypto_op_reset(op, type);
2269
2270         op->phys_addr = rte_mem_virt2iova(_op_data);
2271         op->mempool = mempool;
2272 }
2273
2274
2275 struct rte_mempool *
2276 rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type,
2277                 unsigned nb_elts, unsigned cache_size, uint16_t priv_size,
2278                 int socket_id)
2279 {
2280         struct rte_crypto_op_pool_private *priv;
2281
2282         unsigned elt_size = sizeof(struct rte_crypto_op) +
2283                         priv_size;
2284
2285         if (type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
2286                 elt_size += sizeof(struct rte_crypto_sym_op);
2287         } else if (type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
2288                 elt_size += sizeof(struct rte_crypto_asym_op);
2289         } else if (type == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
2290                 elt_size += RTE_MAX(sizeof(struct rte_crypto_sym_op),
2291                                     sizeof(struct rte_crypto_asym_op));
2292         } else {
2293                 CDEV_LOG_ERR("Invalid op_type\n");
2294                 return NULL;
2295         }
2296
2297         /* lookup mempool in case already allocated */
2298         struct rte_mempool *mp = rte_mempool_lookup(name);
2299
2300         if (mp != NULL) {
2301                 priv = (struct rte_crypto_op_pool_private *)
2302                                 rte_mempool_get_priv(mp);
2303
2304                 if (mp->elt_size != elt_size ||
2305                                 mp->cache_size < cache_size ||
2306                                 mp->size < nb_elts ||
2307                                 priv->priv_size <  priv_size) {
2308                         mp = NULL;
2309                         CDEV_LOG_ERR("Mempool %s already exists but with "
2310                                         "incompatible parameters", name);
2311                         return NULL;
2312                 }
2313                 return mp;
2314         }
2315
2316         mp = rte_mempool_create(
2317                         name,
2318                         nb_elts,
2319                         elt_size,
2320                         cache_size,
2321                         sizeof(struct rte_crypto_op_pool_private),
2322                         NULL,
2323                         NULL,
2324                         rte_crypto_op_init,
2325                         &type,
2326                         socket_id,
2327                         0);
2328
2329         if (mp == NULL) {
2330                 CDEV_LOG_ERR("Failed to create mempool %s", name);
2331                 return NULL;
2332         }
2333
2334         priv = (struct rte_crypto_op_pool_private *)
2335                         rte_mempool_get_priv(mp);
2336
2337         priv->priv_size = priv_size;
2338         priv->type = type;
2339
2340         return mp;
2341 }
2342
2343 int
2344 rte_cryptodev_pmd_create_dev_name(char *name, const char *dev_name_prefix)
2345 {
2346         struct rte_cryptodev *dev = NULL;
2347         uint32_t i = 0;
2348
2349         if (name == NULL)
2350                 return -EINVAL;
2351
2352         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
2353                 int ret = snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
2354                                 "%s_%u", dev_name_prefix, i);
2355
2356                 if (ret < 0)
2357                         return ret;
2358
2359                 dev = rte_cryptodev_pmd_get_named_dev(name);
2360                 if (!dev)
2361                         return 0;
2362         }
2363
2364         return -1;
2365 }
2366
2367 TAILQ_HEAD(cryptodev_driver_list, cryptodev_driver);
2368
2369 static struct cryptodev_driver_list cryptodev_driver_list =
2370         TAILQ_HEAD_INITIALIZER(cryptodev_driver_list);
2371
2372 int
2373 rte_cryptodev_driver_id_get(const char *name)
2374 {
2375         struct cryptodev_driver *driver;
2376         const char *driver_name;
2377
2378         if (name == NULL) {
2379                 RTE_LOG(DEBUG, CRYPTODEV, "name pointer NULL");
2380                 return -1;
2381         }
2382
2383         TAILQ_FOREACH(driver, &cryptodev_driver_list, next) {
2384                 driver_name = driver->driver->name;
2385                 if (strncmp(driver_name, name, strlen(driver_name) + 1) == 0)
2386                         return driver->id;
2387         }
2388         return -1;
2389 }
2390
2391 const char *
2392 rte_cryptodev_name_get(uint8_t dev_id)
2393 {
2394         struct rte_cryptodev *dev;
2395
2396         if (!rte_cryptodev_is_valid_device_data(dev_id)) {
2397                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
2398                 return NULL;
2399         }
2400
2401         dev = rte_cryptodev_pmd_get_dev(dev_id);
2402         if (dev == NULL)
2403                 return NULL;
2404
2405         return dev->data->name;
2406 }
2407
2408 const char *
2409 rte_cryptodev_driver_name_get(uint8_t driver_id)
2410 {
2411         struct cryptodev_driver *driver;
2412
2413         TAILQ_FOREACH(driver, &cryptodev_driver_list, next)
2414                 if (driver->id == driver_id)
2415                         return driver->driver->name;
2416         return NULL;
2417 }
2418
2419 uint8_t
2420 rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv,
2421                 const struct rte_driver *drv)
2422 {
2423         crypto_drv->driver = drv;
2424         crypto_drv->id = nb_drivers;
2425
2426         TAILQ_INSERT_TAIL(&cryptodev_driver_list, crypto_drv, next);
2427
2428         return nb_drivers++;
2429 }