lib/librte_cryptodev/rte_cryptodev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2020 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdint.h>
  14 #include <inttypes.h>
  15 #include <netinet/in.h>
  16
  17 #include <rte_byteorder.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <rte_dev.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_memory.h>
  23 #include <rte_memcpy.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_tailq.h>
  27 #include <rte_eal.h>
  28 #include <rte_per_lcore.h>
  29 #include <rte_lcore.h>
  30 #include <rte_atomic.h>
  31 #include <rte_branch_prediction.h>
  32 #include <rte_common.h>
  33 #include <rte_mempool.h>
  34 #include <rte_malloc.h>
  35 #include <rte_mbuf.h>
  36 #include <rte_errno.h>
  37 #include <rte_spinlock.h>
  38 #include <rte_string_fns.h>
  39
  40 #include "rte_crypto.h"
  41 #include "rte_cryptodev.h"
  42 #include "rte_cryptodev_pmd.h"
  43 #include "rte_cryptodev_trace.h"
  44
  45 static uint8_t nb_drivers;
  46
  47 static struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
  48
  49 struct rte_cryptodev *rte_cryptodevs = rte_crypto_devices;
  50
  51 static struct rte_cryptodev_global cryptodev_globals = {
  52                 .devs                   = rte_crypto_devices,
  53                 .data                   = { NULL },
  54                 .nb_devs                = 0
  55 };
  56
  57 /* spinlock for crypto device callbacks */
  58 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  59
  60 /**
  61  * The user application callback description.
  62  *
  63  * It contains callback address to be registered by user application,
  64  * the pointer to the parameters for callback, and the event type.
  65  */
  66 struct rte_cryptodev_callback {
  67         TAILQ_ENTRY(rte_cryptodev_callback) next; /**< Callbacks list */
  68         rte_cryptodev_cb_fn cb_fn;              /**< Callback address */
  69         void *cb_arg;                           /**< Parameter for callback */
  70         enum rte_cryptodev_event_type event;    /**< Interrupt event type */
  71         uint32_t active;                        /**< Callback is executing */
  72 };
  73
  74 /**
  75  * The crypto cipher algorithm strings identifiers.
  76  * It could be used in application command line.
  77  */
  78 const char *
  79 rte_crypto_cipher_algorithm_strings[] = {
  80         [RTE_CRYPTO_CIPHER_3DES_CBC]    = "3des-cbc",
  81         [RTE_CRYPTO_CIPHER_3DES_ECB]    = "3des-ecb",
  82         [RTE_CRYPTO_CIPHER_3DES_CTR]    = "3des-ctr",
  83
  84         [RTE_CRYPTO_CIPHER_AES_CBC]     = "aes-cbc",
  85         [RTE_CRYPTO_CIPHER_AES_CTR]     = "aes-ctr",
  86         [RTE_CRYPTO_CIPHER_AES_DOCSISBPI]       = "aes-docsisbpi",
  87         [RTE_CRYPTO_CIPHER_AES_ECB]     = "aes-ecb",
  88         [RTE_CRYPTO_CIPHER_AES_F8]      = "aes-f8",
  89         [RTE_CRYPTO_CIPHER_AES_XTS]     = "aes-xts",
  90
  91         [RTE_CRYPTO_CIPHER_ARC4]        = "arc4",
  92
  93         [RTE_CRYPTO_CIPHER_DES_CBC]     = "des-cbc",
  94         [RTE_CRYPTO_CIPHER_DES_DOCSISBPI]       = "des-docsisbpi",
  95
  96         [RTE_CRYPTO_CIPHER_NULL]        = "null",
  97
  98         [RTE_CRYPTO_CIPHER_KASUMI_F8]   = "kasumi-f8",
  99         [RTE_CRYPTO_CIPHER_SNOW3G_UEA2] = "snow3g-uea2",
 100         [RTE_CRYPTO_CIPHER_ZUC_EEA3]    = "zuc-eea3"
 101 };
 102
 103 /**
 104  * The crypto cipher operation strings identifiers.
 105  * It could be used in application command line.
 106  */
 107 const char *
 108 rte_crypto_cipher_operation_strings[] = {
 109                 [RTE_CRYPTO_CIPHER_OP_ENCRYPT]  = "encrypt",
 110                 [RTE_CRYPTO_CIPHER_OP_DECRYPT]  = "decrypt"
 111 };
 112
 113 /**
 114  * The crypto auth algorithm strings identifiers.
 115  * It could be used in application command line.
 116  */
 117 const char *
 118 rte_crypto_auth_algorithm_strings[] = {
 119         [RTE_CRYPTO_AUTH_AES_CBC_MAC]   = "aes-cbc-mac",
 120         [RTE_CRYPTO_AUTH_AES_CMAC]      = "aes-cmac",
 121         [RTE_CRYPTO_AUTH_AES_GMAC]      = "aes-gmac",
 122         [RTE_CRYPTO_AUTH_AES_XCBC_MAC]  = "aes-xcbc-mac",
 123
 124         [RTE_CRYPTO_AUTH_MD5]           = "md5",
 125         [RTE_CRYPTO_AUTH_MD5_HMAC]      = "md5-hmac",
 126
 127         [RTE_CRYPTO_AUTH_NULL]          = "null",
 128
 129         [RTE_CRYPTO_AUTH_SHA1]          = "sha1",
 130         [RTE_CRYPTO_AUTH_SHA1_HMAC]     = "sha1-hmac",
 131
 132         [RTE_CRYPTO_AUTH_SHA224]        = "sha2-224",
 133         [RTE_CRYPTO_AUTH_SHA224_HMAC]   = "sha2-224-hmac",
 134         [RTE_CRYPTO_AUTH_SHA256]        = "sha2-256",
 135         [RTE_CRYPTO_AUTH_SHA256_HMAC]   = "sha2-256-hmac",
 136         [RTE_CRYPTO_AUTH_SHA384]        = "sha2-384",
 137         [RTE_CRYPTO_AUTH_SHA384_HMAC]   = "sha2-384-hmac",
 138         [RTE_CRYPTO_AUTH_SHA512]        = "sha2-512",
 139         [RTE_CRYPTO_AUTH_SHA512_HMAC]   = "sha2-512-hmac",
 140
 141         [RTE_CRYPTO_AUTH_KASUMI_F9]     = "kasumi-f9",
 142         [RTE_CRYPTO_AUTH_SNOW3G_UIA2]   = "snow3g-uia2",
 143         [RTE_CRYPTO_AUTH_ZUC_EIA3]      = "zuc-eia3"
 144 };
 145
 146 /**
 147  * The crypto AEAD algorithm strings identifiers.
 148  * It could be used in application command line.
 149  */
 150 const char *
 151 rte_crypto_aead_algorithm_strings[] = {
 152         [RTE_CRYPTO_AEAD_AES_CCM]       = "aes-ccm",
 153         [RTE_CRYPTO_AEAD_AES_GCM]       = "aes-gcm",
 154         [RTE_CRYPTO_AEAD_CHACHA20_POLY1305] = "chacha20-poly1305"
 155 };
 156
 157 /**
 158  * The crypto AEAD operation strings identifiers.
 159  * It could be used in application command line.
 160  */
 161 const char *
 162 rte_crypto_aead_operation_strings[] = {
 163         [RTE_CRYPTO_AEAD_OP_ENCRYPT]    = "encrypt",
 164         [RTE_CRYPTO_AEAD_OP_DECRYPT]    = "decrypt"
 165 };
 166
 167 /**
 168  * Asymmetric crypto transform operation strings identifiers.
 169  */
 170 const char *rte_crypto_asym_xform_strings[] = {
 171         [RTE_CRYPTO_ASYM_XFORM_NONE]    = "none",
 172         [RTE_CRYPTO_ASYM_XFORM_RSA]     = "rsa",
 173         [RTE_CRYPTO_ASYM_XFORM_MODEX]   = "modexp",
 174         [RTE_CRYPTO_ASYM_XFORM_MODINV]  = "modinv",
 175         [RTE_CRYPTO_ASYM_XFORM_DH]      = "dh",
 176         [RTE_CRYPTO_ASYM_XFORM_DSA]     = "dsa",
 177         [RTE_CRYPTO_ASYM_XFORM_ECDSA]   = "ecdsa",
 178         [RTE_CRYPTO_ASYM_XFORM_ECPM]    = "ecpm",
 179 };
 180
 181 /**
 182  * Asymmetric crypto operation strings identifiers.
 183  */
 184 const char *rte_crypto_asym_op_strings[] = {
 185         [RTE_CRYPTO_ASYM_OP_ENCRYPT]    = "encrypt",
 186         [RTE_CRYPTO_ASYM_OP_DECRYPT]    = "decrypt",
 187         [RTE_CRYPTO_ASYM_OP_SIGN]       = "sign",
 188         [RTE_CRYPTO_ASYM_OP_VERIFY]     = "verify",
 189         [RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE]       = "priv_key_generate",
 190         [RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE] = "pub_key_generate",
 191         [RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE] = "sharedsecret_compute",
 192 };
 193
 194 /**
 195  * The private data structure stored in the session mempool private data.
 196  */
 197 struct rte_cryptodev_sym_session_pool_private_data {
 198         uint16_t nb_drivers;
 199         /**< number of elements in sess_data array */
 200         uint16_t user_data_sz;
 201         /**< session user data will be placed after sess_data */
 202 };
 203
 204 int
 205 rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum,
 206                 const char *algo_string)
 207 {
 208         unsigned int i;
 209
 210         for (i = 1; i < RTE_DIM(rte_crypto_cipher_algorithm_strings); i++) {
 211                 if (strcmp(algo_string, rte_crypto_cipher_algorithm_strings[i]) == 0) {
 212                         *algo_enum = (enum rte_crypto_cipher_algorithm) i;
 213                         return 0;
 214                 }
 215         }
 216
 217         /* Invalid string */
 218         return -1;
 219 }
 220
 221 int
 222 rte_cryptodev_get_auth_algo_enum(enum rte_crypto_auth_algorithm *algo_enum,
 223                 const char *algo_string)
 224 {
 225         unsigned int i;
 226
 227         for (i = 1; i < RTE_DIM(rte_crypto_auth_algorithm_strings); i++) {
 228                 if (strcmp(algo_string, rte_crypto_auth_algorithm_strings[i]) == 0) {
 229                         *algo_enum = (enum rte_crypto_auth_algorithm) i;
 230                         return 0;
 231                 }
 232         }
 233
 234         /* Invalid string */
 235         return -1;
 236 }
 237
 238 int
 239 rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum,
 240                 const char *algo_string)
 241 {
 242         unsigned int i;
 243
 244         for (i = 1; i < RTE_DIM(rte_crypto_aead_algorithm_strings); i++) {
 245                 if (strcmp(algo_string, rte_crypto_aead_algorithm_strings[i]) == 0) {
 246                         *algo_enum = (enum rte_crypto_aead_algorithm) i;
 247                         return 0;
 248                 }
 249         }
 250
 251         /* Invalid string */
 252         return -1;
 253 }
 254
 255 int
 256 rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
 257                 const char *xform_string)
 258 {
 259         unsigned int i;
 260
 261         for (i = 1; i < RTE_DIM(rte_crypto_asym_xform_strings); i++) {
 262                 if (strcmp(xform_string,
 263                         rte_crypto_asym_xform_strings[i]) == 0) {
 264                         *xform_enum = (enum rte_crypto_asym_xform_type) i;
 265                         return 0;
 266                 }
 267         }
 268
 269         /* Invalid string */
 270         return -1;
 271 }
 272
 273 /**
 274  * The crypto auth operation strings identifiers.
 275  * It could be used in application command line.
 276  */
 277 const char *
 278 rte_crypto_auth_operation_strings[] = {
 279                 [RTE_CRYPTO_AUTH_OP_VERIFY]     = "verify",
 280                 [RTE_CRYPTO_AUTH_OP_GENERATE]   = "generate"
 281 };
 282
 283 const struct rte_cryptodev_symmetric_capability *
 284 rte_cryptodev_sym_capability_get(uint8_t dev_id,
 285                 const struct rte_cryptodev_sym_capability_idx *idx)
 286 {
 287         const struct rte_cryptodev_capabilities *capability;
 288         struct rte_cryptodev_info dev_info;
 289         int i = 0;
 290
 291         rte_cryptodev_info_get(dev_id, &dev_info);
 292
 293         while ((capability = &dev_info.capabilities[i++])->op !=
 294                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 295                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
 296                         continue;
 297
 298                 if (capability->sym.xform_type != idx->type)
 299                         continue;
 300
 301                 if (idx->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 302                         capability->sym.auth.algo == idx->algo.auth)
 303                         return &capability->sym;
 304
 305                 if (idx->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 306                         capability->sym.cipher.algo == idx->algo.cipher)
 307                         return &capability->sym;
 308
 309                 if (idx->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
 310                                 capability->sym.aead.algo == idx->algo.aead)
 311                         return &capability->sym;
 312         }
 313
 314         return NULL;
 315 }
 316
 317 static int
 318 param_range_check(uint16_t size, const struct rte_crypto_param_range *range)
 319 {
 320         unsigned int next_size;
 321
 322         /* Check lower/upper bounds */
 323         if (size < range->min)
 324                 return -1;
 325
 326         if (size > range->max)
 327                 return -1;
 328
 329         /* If range is actually only one value, size is correct */
 330         if (range->increment == 0)
 331                 return 0;
 332
 333         /* Check if value is one of the supported sizes */
 334         for (next_size = range->min; next_size <= range->max;
 335                         next_size += range->increment)
 336                 if (size == next_size)
 337                         return 0;
 338
 339         return -1;
 340 }
 341
 342 const struct rte_cryptodev_asymmetric_xform_capability *
 343 rte_cryptodev_asym_capability_get(uint8_t dev_id,
 344                 const struct rte_cryptodev_asym_capability_idx *idx)
 345 {
 346         const struct rte_cryptodev_capabilities *capability;
 347         struct rte_cryptodev_info dev_info;
 348         unsigned int i = 0;
 349
 350         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 351         rte_cryptodev_info_get(dev_id, &dev_info);
 352
 353         while ((capability = &dev_info.capabilities[i++])->op !=
 354                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 355                 if (capability->op != RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
 356                         continue;
 357
 358                 if (capability->asym.xform_capa.xform_type == idx->type)
 359                         return &capability->asym.xform_capa;
 360         }
 361         return NULL;
 362 };
 363
 364 int
 365 rte_cryptodev_sym_capability_check_cipher(
 366                 const struct rte_cryptodev_symmetric_capability *capability,
 367                 uint16_t key_size, uint16_t iv_size)
 368 {
 369         if (param_range_check(key_size, &capability->cipher.key_size) != 0)
 370                 return -1;
 371
 372         if (param_range_check(iv_size, &capability->cipher.iv_size) != 0)
 373                 return -1;
 374
 375         return 0;
 376 }
 377
 378 int
 379 rte_cryptodev_sym_capability_check_auth(
 380                 const struct rte_cryptodev_symmetric_capability *capability,
 381                 uint16_t key_size, uint16_t digest_size, uint16_t iv_size)
 382 {
 383         if (param_range_check(key_size, &capability->auth.key_size) != 0)
 384                 return -1;
 385
 386         if (param_range_check(digest_size, &capability->auth.digest_size) != 0)
 387                 return -1;
 388
 389         if (param_range_check(iv_size, &capability->auth.iv_size) != 0)
 390                 return -1;
 391
 392         return 0;
 393 }
 394
 395 int
 396 rte_cryptodev_sym_capability_check_aead(
 397                 const struct rte_cryptodev_symmetric_capability *capability,
 398                 uint16_t key_size, uint16_t digest_size, uint16_t aad_size,
 399                 uint16_t iv_size)
 400 {
 401         if (param_range_check(key_size, &capability->aead.key_size) != 0)
 402                 return -1;
 403
 404         if (param_range_check(digest_size, &capability->aead.digest_size) != 0)
 405                 return -1;
 406
 407         if (param_range_check(aad_size, &capability->aead.aad_size) != 0)
 408                 return -1;
 409
 410         if (param_range_check(iv_size, &capability->aead.iv_size) != 0)
 411                 return -1;
 412
 413         return 0;
 414 }
 415 int
 416 rte_cryptodev_asym_xform_capability_check_optype(
 417         const struct rte_cryptodev_asymmetric_xform_capability *capability,
 418         enum rte_crypto_asym_op_type op_type)
 419 {
 420         if (capability->op_types & (1 << op_type))
 421                 return 1;
 422
 423         return 0;
 424 }
 425
 426 int
 427 rte_cryptodev_asym_xform_capability_check_modlen(
 428         const struct rte_cryptodev_asymmetric_xform_capability *capability,
 429         uint16_t modlen)
 430 {
 431         /* no need to check for limits, if min or max = 0 */
 432         if (capability->modlen.min != 0) {
 433                 if (modlen < capability->modlen.min)
 434                         return -1;
 435         }
 436
 437         if (capability->modlen.max != 0) {
 438                 if (modlen > capability->modlen.max)
 439                         return -1;
 440         }
 441
 442         /* in any case, check if given modlen is module increment */
 443         if (capability->modlen.increment != 0) {
 444                 if (modlen % (capability->modlen.increment))
 445                         return -1;
 446         }
 447
 448         return 0;
 449 }
 450
 451
 452 const char *
 453 rte_cryptodev_get_feature_name(uint64_t flag)
 454 {
 455         switch (flag) {
 456         case RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO:
 457                 return "SYMMETRIC_CRYPTO";
 458         case RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO:
 459                 return "ASYMMETRIC_CRYPTO";
 460         case RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING:
 461                 return "SYM_OPERATION_CHAINING";
 462         case RTE_CRYPTODEV_FF_CPU_SSE:
 463                 return "CPU_SSE";
 464         case RTE_CRYPTODEV_FF_CPU_AVX:
 465                 return "CPU_AVX";
 466         case RTE_CRYPTODEV_FF_CPU_AVX2:
 467                 return "CPU_AVX2";
 468         case RTE_CRYPTODEV_FF_CPU_AVX512:
 469                 return "CPU_AVX512";
 470         case RTE_CRYPTODEV_FF_CPU_AESNI:
 471                 return "CPU_AESNI";
 472         case RTE_CRYPTODEV_FF_HW_ACCELERATED:
 473                 return "HW_ACCELERATED";
 474         case RTE_CRYPTODEV_FF_IN_PLACE_SGL:
 475                 return "IN_PLACE_SGL";
 476         case RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT:
 477                 return "OOP_SGL_IN_SGL_OUT";
 478         case RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT:
 479                 return "OOP_SGL_IN_LB_OUT";
 480         case RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT:
 481                 return "OOP_LB_IN_SGL_OUT";
 482         case RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT:
 483                 return "OOP_LB_IN_LB_OUT";
 484         case RTE_CRYPTODEV_FF_CPU_NEON:
 485                 return "CPU_NEON";
 486         case RTE_CRYPTODEV_FF_CPU_ARM_CE:
 487                 return "CPU_ARM_CE";
 488         case RTE_CRYPTODEV_FF_SECURITY:
 489                 return "SECURITY_PROTOCOL";
 490         case RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP:
 491                 return "RSA_PRIV_OP_KEY_EXP";
 492         case RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT:
 493                 return "RSA_PRIV_OP_KEY_QT";
 494         case RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED:
 495                 return "DIGEST_ENCRYPTED";
 496         case RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO:
 497                 return "SYM_CPU_CRYPTO";
 498         case RTE_CRYPTODEV_FF_ASYM_SESSIONLESS:
 499                 return "ASYM_SESSIONLESS";
 500         case RTE_CRYPTODEV_FF_SYM_SESSIONLESS:
 501                 return "SYM_SESSIONLESS";
 502         case RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA:
 503                 return "NON_BYTE_ALIGNED_DATA";
 504         default:
 505                 return NULL;
 506         }
 507 }
 508
 509 struct rte_cryptodev *
 510 rte_cryptodev_pmd_get_dev(uint8_t dev_id)
 511 {
 512         return &cryptodev_globals.devs[dev_id];
 513 }
 514
 515 struct rte_cryptodev *
 516 rte_cryptodev_pmd_get_named_dev(const char *name)
 517 {
 518         struct rte_cryptodev *dev;
 519         unsigned int i;
 520
 521         if (name == NULL)
 522                 return NULL;
 523
 524         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
 525                 dev = &cryptodev_globals.devs[i];
 526
 527                 if ((dev->attached == RTE_CRYPTODEV_ATTACHED) &&
 528                                 (strcmp(dev->data->name, name) == 0))
 529                         return dev;
 530         }
 531
 532         return NULL;
 533 }
 534
 535 static inline uint8_t
 536 rte_cryptodev_is_valid_device_data(uint8_t dev_id)
 537 {
 538         if (dev_id >= RTE_CRYPTO_MAX_DEVS ||
 539                         rte_crypto_devices[dev_id].data == NULL)
 540                 return 0;
 541
 542         return 1;
 543 }
 544
 545 unsigned int
 546 rte_cryptodev_pmd_is_valid_dev(uint8_t dev_id)
 547 {
 548         struct rte_cryptodev *dev = NULL;
 549
 550         if (!rte_cryptodev_is_valid_device_data(dev_id))
 551                 return 0;
 552
 553         dev = rte_cryptodev_pmd_get_dev(dev_id);
 554         if (dev->attached != RTE_CRYPTODEV_ATTACHED)
 555                 return 0;
 556         else
 557                 return 1;
 558 }
 559
 560
 561 int
 562 rte_cryptodev_get_dev_id(const char *name)
 563 {
 564         unsigned i;
 565
 566         if (name == NULL)
 567                 return -1;
 568
 569         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
 570                 if (!rte_cryptodev_is_valid_device_data(i))
 571                         continue;
 572                 if ((strcmp(cryptodev_globals.devs[i].data->name, name)
 573                                 == 0) &&
 574                                 (cryptodev_globals.devs[i].attached ==
 575                                                 RTE_CRYPTODEV_ATTACHED))
 576                         return i;
 577         }
 578
 579         return -1;
 580 }
 581
 582 uint8_t
 583 rte_cryptodev_count(void)
 584 {
 585         return cryptodev_globals.nb_devs;
 586 }
 587
 588 uint8_t
 589 rte_cryptodev_device_count_by_driver(uint8_t driver_id)
 590 {
 591         uint8_t i, dev_count = 0;
 592
 593         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++)
 594                 if (cryptodev_globals.devs[i].driver_id == driver_id &&
 595                         cryptodev_globals.devs[i].attached ==
 596                                         RTE_CRYPTODEV_ATTACHED)
 597                         dev_count++;
 598
 599         return dev_count;
 600 }
 601
 602 uint8_t
 603 rte_cryptodev_devices_get(const char *driver_name, uint8_t *devices,
 604         uint8_t nb_devices)
 605 {
 606         uint8_t i, count = 0;
 607         struct rte_cryptodev *devs = cryptodev_globals.devs;
 608
 609         for (i = 0; i < RTE_CRYPTO_MAX_DEVS && count < nb_devices; i++) {
 610                 if (!rte_cryptodev_is_valid_device_data(i))
 611                         continue;
 612
 613                 if (devs[i].attached == RTE_CRYPTODEV_ATTACHED) {
 614                         int cmp;
 615
 616                         cmp = strncmp(devs[i].device->driver->name,
 617                                         driver_name,
 618                                         strlen(driver_name) + 1);
 619
 620                         if (cmp == 0)
 621                                 devices[count++] = devs[i].data->dev_id;
 622                 }
 623         }
 624
 625         return count;
 626 }
 627
 628 void *
 629 rte_cryptodev_get_sec_ctx(uint8_t dev_id)
 630 {
 631         if (dev_id < RTE_CRYPTO_MAX_DEVS &&
 632                         (rte_crypto_devices[dev_id].feature_flags &
 633                         RTE_CRYPTODEV_FF_SECURITY))
 634                 return rte_crypto_devices[dev_id].security_ctx;
 635
 636         return NULL;
 637 }
 638
 639 int
 640 rte_cryptodev_socket_id(uint8_t dev_id)
 641 {
 642         struct rte_cryptodev *dev;
 643
 644         if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
 645                 return -1;
 646
 647         dev = rte_cryptodev_pmd_get_dev(dev_id);
 648
 649         return dev->data->socket_id;
 650 }
 651
 652 static inline int
 653 rte_cryptodev_data_alloc(uint8_t dev_id, struct rte_cryptodev_data **data,
 654                 int socket_id)
 655 {
 656         char mz_name[RTE_MEMZONE_NAMESIZE];
 657         const struct rte_memzone *mz;
 658         int n;
 659
 660         /* generate memzone name */
 661         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 662         if (n >= (int)sizeof(mz_name))
 663                 return -EINVAL;
 664
 665         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 666                 mz = rte_memzone_reserve(mz_name,
 667                                 sizeof(struct rte_cryptodev_data),
 668                                 socket_id, 0);
 669                 CDEV_LOG_DEBUG("PRIMARY:reserved memzone for %s (%p)",
 670                                 mz_name, mz);
 671         } else {
 672                 mz = rte_memzone_lookup(mz_name);
 673                 CDEV_LOG_DEBUG("SECONDARY:looked up memzone for %s (%p)",
 674                                 mz_name, mz);
 675         }
 676
 677         if (mz == NULL)
 678                 return -ENOMEM;
 679
 680         *data = mz->addr;
 681         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 682                 memset(*data, 0, sizeof(struct rte_cryptodev_data));
 683
 684         return 0;
 685 }
 686
 687 static inline int
 688 rte_cryptodev_data_free(uint8_t dev_id, struct rte_cryptodev_data **data)
 689 {
 690         char mz_name[RTE_MEMZONE_NAMESIZE];
 691         const struct rte_memzone *mz;
 692         int n;
 693
 694         /* generate memzone name */
 695         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 696         if (n >= (int)sizeof(mz_name))
 697                 return -EINVAL;
 698
 699         mz = rte_memzone_lookup(mz_name);
 700         if (mz == NULL)
 701                 return -ENOMEM;
 702
 703         RTE_ASSERT(*data == mz->addr);
 704         *data = NULL;
 705
 706         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 707                 CDEV_LOG_DEBUG("PRIMARY:free memzone of %s (%p)",
 708                                 mz_name, mz);
 709                 return rte_memzone_free(mz);
 710         } else {
 711                 CDEV_LOG_DEBUG("SECONDARY:don't free memzone of %s (%p)",
 712                                 mz_name, mz);
 713         }
 714
 715         return 0;
 716 }
 717
 718 static uint8_t
 719 rte_cryptodev_find_free_device_index(void)
 720 {
 721         uint8_t dev_id;
 722
 723         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++) {
 724                 if (rte_crypto_devices[dev_id].attached ==
 725                                 RTE_CRYPTODEV_DETACHED)
 726                         return dev_id;
 727         }
 728         return RTE_CRYPTO_MAX_DEVS;
 729 }
 730
 731 struct rte_cryptodev *
 732 rte_cryptodev_pmd_allocate(const char *name, int socket_id)
 733 {
 734         struct rte_cryptodev *cryptodev;
 735         uint8_t dev_id;
 736
 737         if (rte_cryptodev_pmd_get_named_dev(name) != NULL) {
 738                 CDEV_LOG_ERR("Crypto device with name %s already "
 739                                 "allocated!", name);
 740                 return NULL;
 741         }
 742
 743         dev_id = rte_cryptodev_find_free_device_index();
 744         if (dev_id == RTE_CRYPTO_MAX_DEVS) {
 745                 CDEV_LOG_ERR("Reached maximum number of crypto devices");
 746                 return NULL;
 747         }
 748
 749         cryptodev = rte_cryptodev_pmd_get_dev(dev_id);
 750
 751         if (cryptodev->data == NULL) {
 752                 struct rte_cryptodev_data **cryptodev_data =
 753                                 &cryptodev_globals.data[dev_id];
 754
 755                 int retval = rte_cryptodev_data_alloc(dev_id, cryptodev_data,
 756                                 socket_id);
 757
 758                 if (retval < 0 || *cryptodev_data == NULL)
 759                         return NULL;
 760
 761                 cryptodev->data = *cryptodev_data;
 762
 763                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 764                         strlcpy(cryptodev->data->name, name,
 765                                 RTE_CRYPTODEV_NAME_MAX_LEN);
 766
 767                         cryptodev->data->dev_id = dev_id;
 768                         cryptodev->data->socket_id = socket_id;
 769                         cryptodev->data->dev_started = 0;
 770                         CDEV_LOG_DEBUG("PRIMARY:init data");
 771                 }
 772
 773                 CDEV_LOG_DEBUG("Data for %s: dev_id %d, socket %d, started %d",
 774                                 cryptodev->data->name,
 775                                 cryptodev->data->dev_id,
 776                                 cryptodev->data->socket_id,
 777                                 cryptodev->data->dev_started);
 778
 779                 /* init user callbacks */
 780                 TAILQ_INIT(&(cryptodev->link_intr_cbs));
 781
 782                 cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 783
 784                 cryptodev_globals.nb_devs++;
 785         }
 786
 787         return cryptodev;
 788 }
 789
 790 int
 791 rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 792 {
 793         int ret;
 794         uint8_t dev_id;
 795
 796         if (cryptodev == NULL)
 797                 return -EINVAL;
 798
 799         dev_id = cryptodev->data->dev_id;
 800
 801         /* Close device only if device operations have been set */
 802         if (cryptodev->dev_ops) {
 803                 ret = rte_cryptodev_close(dev_id);
 804                 if (ret < 0)
 805                         return ret;
 806         }
 807
 808         ret = rte_cryptodev_data_free(dev_id, &cryptodev_globals.data[dev_id]);
 809         if (ret < 0)
 810                 return ret;
 811
 812         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 813         cryptodev_globals.nb_devs--;
 814         return 0;
 815 }
 816
 817 uint16_t
 818 rte_cryptodev_queue_pair_count(uint8_t dev_id)
 819 {
 820         struct rte_cryptodev *dev;
 821
 822         if (!rte_cryptodev_is_valid_device_data(dev_id)) {
 823                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 824                 return 0;
 825         }
 826
 827         dev = &rte_crypto_devices[dev_id];
 828         return dev->data->nb_queue_pairs;
 829 }
 830
 831 static int
 832 rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 833                 int socket_id)
 834 {
 835         struct rte_cryptodev_info dev_info;
 836         void **qp;
 837         unsigned i;
 838
 839         if ((dev == NULL) || (nb_qpairs < 1)) {
 840                 CDEV_LOG_ERR("invalid param: dev %p, nb_queues %u",
 841                                                         dev, nb_qpairs);
 842                 return -EINVAL;
 843         }
 844
 845         CDEV_LOG_DEBUG("Setup %d queues pairs on device %u",
 846                         nb_qpairs, dev->data->dev_id);
 847
 848         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 849
 850         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 851         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 852
 853         if (nb_qpairs > (dev_info.max_nb_queue_pairs)) {
 854                 CDEV_LOG_ERR("Invalid num queue_pairs (%u) for dev %u",
 855                                 nb_qpairs, dev->data->dev_id);
 856             return -EINVAL;
 857         }
 858
 859         if (dev->data->queue_pairs == NULL) { /* first time configuration */
 860                 dev->data->queue_pairs = rte_zmalloc_socket(
 861                                 "cryptodev->queue_pairs",
 862                                 sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
 863                                 RTE_CACHE_LINE_SIZE, socket_id);
 864
 865                 if (dev->data->queue_pairs == NULL) {
 866                         dev->data->nb_queue_pairs = 0;
 867                         CDEV_LOG_ERR("failed to get memory for qp meta data, "
 868                                                         "nb_queues %u",
 869                                                         nb_qpairs);
 870                         return -(ENOMEM);
 871                 }
 872         } else { /* re-configure */
 873                 int ret;
 874                 uint16_t old_nb_queues = dev->data->nb_queue_pairs;
 875
 876                 qp = dev->data->queue_pairs;
 877
 878                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
 879                                 -ENOTSUP);
 880
 881                 for (i = nb_qpairs; i < old_nb_queues; i++) {
 882                         ret = (*dev->dev_ops->queue_pair_release)(dev, i);
 883                         if (ret < 0)
 884                                 return ret;
 885                 }
 886
 887                 qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
 888                                 RTE_CACHE_LINE_SIZE);
 889                 if (qp == NULL) {
 890                         CDEV_LOG_ERR("failed to realloc qp meta data,"
 891                                                 " nb_queues %u", nb_qpairs);
 892                         return -(ENOMEM);
 893                 }
 894
 895                 if (nb_qpairs > old_nb_queues) {
 896                         uint16_t new_qs = nb_qpairs - old_nb_queues;
 897
 898                         memset(qp + old_nb_queues, 0,
 899                                 sizeof(qp[0]) * new_qs);
 900                 }
 901
 902                 dev->data->queue_pairs = qp;
 903
 904         }
 905         dev->data->nb_queue_pairs = nb_qpairs;
 906         return 0;
 907 }
 908
 909 int
 910 rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
 911 {
 912         struct rte_cryptodev *dev;
 913         int diag;
 914
 915         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 916                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 917                 return -EINVAL;
 918         }
 919
 920         dev = &rte_crypto_devices[dev_id];
 921
 922         if (dev->data->dev_started) {
 923                 CDEV_LOG_ERR(
 924                     "device %d must be stopped to allow configuration", dev_id);
 925                 return -EBUSY;
 926         }
 927
 928         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
 929
 930         /* Setup new number of queue pairs and reconfigure device. */
 931         diag = rte_cryptodev_queue_pairs_config(dev, config->nb_queue_pairs,
 932                         config->socket_id);
 933         if (diag != 0) {
 934                 CDEV_LOG_ERR("dev%d rte_crypto_dev_queue_pairs_config = %d",
 935                                 dev_id, diag);
 936                 return diag;
 937         }
 938
 939         rte_cryptodev_trace_configure(dev_id, config);
 940         return (*dev->dev_ops->dev_configure)(dev, config);
 941 }
 942
 943
 944 int
 945 rte_cryptodev_start(uint8_t dev_id)
 946 {
 947         struct rte_cryptodev *dev;
 948         int diag;
 949
 950         CDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id);
 951
 952         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 953                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 954                 return -EINVAL;
 955         }
 956
 957         dev = &rte_crypto_devices[dev_id];
 958
 959         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
 960
 961         if (dev->data->dev_started != 0) {
 962                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already started",
 963                         dev_id);
 964                 return 0;
 965         }
 966
 967         diag = (*dev->dev_ops->dev_start)(dev);
 968         rte_cryptodev_trace_start(dev_id, diag);
 969         if (diag == 0)
 970                 dev->data->dev_started = 1;
 971         else
 972                 return diag;
 973
 974         return 0;
 975 }
 976
 977 void
 978 rte_cryptodev_stop(uint8_t dev_id)
 979 {
 980         struct rte_cryptodev *dev;
 981
 982         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 983                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 984                 return;
 985         }
 986
 987         dev = &rte_crypto_devices[dev_id];
 988
 989         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
 990
 991         if (dev->data->dev_started == 0) {
 992                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already stopped",
 993                         dev_id);
 994                 return;
 995         }
 996
 997         (*dev->dev_ops->dev_stop)(dev);
 998         rte_cryptodev_trace_stop(dev_id);
 999         dev->data->dev_started = 0;
1000 }
1001
1002 int
1003 rte_cryptodev_close(uint8_t dev_id)
1004 {
1005         struct rte_cryptodev *dev;
1006         int retval;
1007
1008         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1009                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1010                 return -1;
1011         }
1012
1013         dev = &rte_crypto_devices[dev_id];
1014
1015         /* Device must be stopped before it can be closed */
1016         if (dev->data->dev_started == 1) {
1017                 CDEV_LOG_ERR("Device %u must be stopped before closing",
1018                                 dev_id);
1019                 return -EBUSY;
1020         }
1021
1022         /* We can't close the device if there are outstanding sessions in use */
1023         if (dev->data->session_pool != NULL) {
1024                 if (!rte_mempool_full(dev->data->session_pool)) {
1025                         CDEV_LOG_ERR("dev_id=%u close failed, session mempool "
1026                                         "has sessions still in use, free "
1027                                         "all sessions before calling close",
1028                                         (unsigned)dev_id);
1029                         return -EBUSY;
1030                 }
1031         }
1032
1033         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
1034         retval = (*dev->dev_ops->dev_close)(dev);
1035         rte_cryptodev_trace_close(dev_id, retval);
1036
1037         if (retval < 0)
1038                 return retval;
1039
1040         return 0;
1041 }
1042
1043 int
1044 rte_cryptodev_get_qp_status(uint8_t dev_id, uint16_t queue_pair_id)
1045 {
1046         struct rte_cryptodev *dev;
1047
1048         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1049                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1050                 return -EINVAL;
1051         }
1052
1053         dev = &rte_crypto_devices[dev_id];
1054         if (queue_pair_id >= dev->data->nb_queue_pairs) {
1055                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
1056                 return -EINVAL;
1057         }
1058         void **qps = dev->data->queue_pairs;
1059
1060         if (qps[queue_pair_id]) {
1061                 CDEV_LOG_DEBUG("qp %d on dev %d is initialised",
1062                         queue_pair_id, dev_id);
1063                 return 1;
1064         }
1065
1066         CDEV_LOG_DEBUG("qp %d on dev %d is not initialised",
1067                 queue_pair_id, dev_id);
1068
1069         return 0;
1070 }
1071
1072 int
1073 rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
1074                 const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
1075
1076 {
1077         struct rte_cryptodev *dev;
1078
1079         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1080                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1081                 return -EINVAL;
1082         }
1083
1084         dev = &rte_crypto_devices[dev_id];
1085         if (queue_pair_id >= dev->data->nb_queue_pairs) {
1086                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
1087                 return -EINVAL;
1088         }
1089
1090         if (!qp_conf) {
1091                 CDEV_LOG_ERR("qp_conf cannot be NULL\n");
1092                 return -EINVAL;
1093         }
1094
1095         if ((qp_conf->mp_session && !qp_conf->mp_session_private) ||
1096                         (!qp_conf->mp_session && qp_conf->mp_session_private)) {
1097                 CDEV_LOG_ERR("Invalid mempools\n");
1098                 return -EINVAL;
1099         }
1100
1101         if (qp_conf->mp_session) {
1102                 struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1103                 uint32_t obj_size = qp_conf->mp_session->elt_size;
1104                 uint32_t obj_priv_size = qp_conf->mp_session_private->elt_size;
1105                 struct rte_cryptodev_sym_session s = {0};
1106
1107                 pool_priv = rte_mempool_get_priv(qp_conf->mp_session);
1108                 if (!pool_priv || qp_conf->mp_session->private_data_size <
1109                                 sizeof(*pool_priv)) {
1110                         CDEV_LOG_ERR("Invalid mempool\n");
1111                         return -EINVAL;
1112                 }
1113
1114                 s.nb_drivers = pool_priv->nb_drivers;
1115                 s.user_data_sz = pool_priv->user_data_sz;
1116
1117                 if ((rte_cryptodev_sym_get_existing_header_session_size(&s) >
1118                         obj_size) || (s.nb_drivers <= dev->driver_id) ||
1119                         rte_cryptodev_sym_get_private_session_size(dev_id) >
1120                                 obj_priv_size) {
1121                         CDEV_LOG_ERR("Invalid mempool\n");
1122                         return -EINVAL;
1123                 }
1124         }
1125
1126         if (dev->data->dev_started) {
1127                 CDEV_LOG_ERR(
1128                     "device %d must be stopped to allow configuration", dev_id);
1129                 return -EBUSY;
1130         }
1131
1132         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
1133
1134         rte_cryptodev_trace_queue_pair_setup(dev_id, queue_pair_id, qp_conf);
1135         return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id, qp_conf,
1136                         socket_id);
1137 }
1138
1139
1140 int
1141 rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats)
1142 {
1143         struct rte_cryptodev *dev;
1144
1145         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1146                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1147                 return -ENODEV;
1148         }
1149
1150         if (stats == NULL) {
1151                 CDEV_LOG_ERR("Invalid stats ptr");
1152                 return -EINVAL;
1153         }
1154
1155         dev = &rte_crypto_devices[dev_id];
1156         memset(stats, 0, sizeof(*stats));
1157
1158         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1159         (*dev->dev_ops->stats_get)(dev, stats);
1160         return 0;
1161 }
1162
1163 void
1164 rte_cryptodev_stats_reset(uint8_t dev_id)
1165 {
1166         struct rte_cryptodev *dev;
1167
1168         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1169                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1170                 return;
1171         }
1172
1173         dev = &rte_crypto_devices[dev_id];
1174
1175         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
1176         (*dev->dev_ops->stats_reset)(dev);
1177 }
1178
1179 void
1180 rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
1181 {
1182         struct rte_cryptodev *dev;
1183
1184         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1185                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1186                 return;
1187         }
1188
1189         dev = &rte_crypto_devices[dev_id];
1190
1191         memset(dev_info, 0, sizeof(struct rte_cryptodev_info));
1192
1193         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
1194         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
1195
1196         dev_info->driver_name = dev->device->driver->name;
1197         dev_info->device = dev->device;
1198 }
1199
1200 int
1201 rte_cryptodev_callback_register(uint8_t dev_id,
1202                         enum rte_cryptodev_event_type event,
1203                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
1204 {
1205         struct rte_cryptodev *dev;
1206         struct rte_cryptodev_callback *user_cb;
1207
1208         if (!cb_fn)
1209                 return -EINVAL;
1210
1211         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1212                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1213                 return -EINVAL;
1214         }
1215
1216         dev = &rte_crypto_devices[dev_id];
1217         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1218
1219         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
1220                 if (user_cb->cb_fn == cb_fn &&
1221                         user_cb->cb_arg == cb_arg &&
1222                         user_cb->event == event) {
1223                         break;
1224                 }
1225         }
1226
1227         /* create a new callback. */
1228         if (user_cb == NULL) {
1229                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
1230                                 sizeof(struct rte_cryptodev_callback), 0);
1231                 if (user_cb != NULL) {
1232                         user_cb->cb_fn = cb_fn;
1233                         user_cb->cb_arg = cb_arg;
1234                         user_cb->event = event;
1235                         TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
1236                 }
1237         }
1238
1239         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1240         return (user_cb == NULL) ? -ENOMEM : 0;
1241 }
1242
1243 int
1244 rte_cryptodev_callback_unregister(uint8_t dev_id,
1245                         enum rte_cryptodev_event_type event,
1246                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
1247 {
1248         int ret;
1249         struct rte_cryptodev *dev;
1250         struct rte_cryptodev_callback *cb, *next;
1251
1252         if (!cb_fn)
1253                 return -EINVAL;
1254
1255         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1256                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1257                 return -EINVAL;
1258         }
1259
1260         dev = &rte_crypto_devices[dev_id];
1261         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1262
1263         ret = 0;
1264         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
1265
1266                 next = TAILQ_NEXT(cb, next);
1267
1268                 if (cb->cb_fn != cb_fn || cb->event != event ||
1269                                 (cb->cb_arg != (void *)-1 &&
1270                                 cb->cb_arg != cb_arg))
1271                         continue;
1272
1273                 /*
1274                  * if this callback is not executing right now,
1275                  * then remove it.
1276                  */
1277                 if (cb->active == 0) {
1278                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
1279                         rte_free(cb);
1280                 } else {
1281                         ret = -EAGAIN;
1282                 }
1283         }
1284
1285         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1286         return ret;
1287 }
1288
1289 void
1290 rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev,
1291         enum rte_cryptodev_event_type event)
1292 {
1293         struct rte_cryptodev_callback *cb_lst;
1294         struct rte_cryptodev_callback dev_cb;
1295
1296         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1297         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
1298                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1299                         continue;
1300                 dev_cb = *cb_lst;
1301                 cb_lst->active = 1;
1302                 rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1303                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1304                                                 dev_cb.cb_arg);
1305                 rte_spinlock_lock(&rte_cryptodev_cb_lock);
1306                 cb_lst->active = 0;
1307         }
1308         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1309 }
1310
1311 int
1312 rte_cryptodev_sym_session_init(uint8_t dev_id,
1313                 struct rte_cryptodev_sym_session *sess,
1314                 struct rte_crypto_sym_xform *xforms,
1315                 struct rte_mempool *mp)
1316 {
1317         struct rte_cryptodev *dev;
1318         uint32_t sess_priv_sz = rte_cryptodev_sym_get_private_session_size(
1319                         dev_id);
1320         uint8_t index;
1321         int ret;
1322
1323         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1324                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1325                 return -EINVAL;
1326         }
1327
1328         dev = rte_cryptodev_pmd_get_dev(dev_id);
1329
1330         if (sess == NULL || xforms == NULL || dev == NULL || mp == NULL)
1331                 return -EINVAL;
1332
1333         if (mp->elt_size < sess_priv_sz)
1334                 return -EINVAL;
1335
1336         index = dev->driver_id;
1337         if (index >= sess->nb_drivers)
1338                 return -EINVAL;
1339
1340         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_configure, -ENOTSUP);
1341
1342         if (sess->sess_data[index].refcnt == 0) {
1343                 ret = dev->dev_ops->sym_session_configure(dev, xforms,
1344                                                         sess, mp);
1345                 if (ret < 0) {
1346                         CDEV_LOG_ERR(
1347                                 "dev_id %d failed to configure session details",
1348                                 dev_id);
1349                         return ret;
1350                 }
1351         }
1352
1353         rte_cryptodev_trace_sym_session_init(dev_id, sess, xforms, mp);
1354         sess->sess_data[index].refcnt++;
1355         return 0;
1356 }
1357
1358 int
1359 rte_cryptodev_asym_session_init(uint8_t dev_id,
1360                 struct rte_cryptodev_asym_session *sess,
1361                 struct rte_crypto_asym_xform *xforms,
1362                 struct rte_mempool *mp)
1363 {
1364         struct rte_cryptodev *dev;
1365         uint8_t index;
1366         int ret;
1367
1368         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1369                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1370                 return -EINVAL;
1371         }
1372
1373         dev = rte_cryptodev_pmd_get_dev(dev_id);
1374
1375         if (sess == NULL || xforms == NULL || dev == NULL)
1376                 return -EINVAL;
1377
1378         index = dev->driver_id;
1379
1380         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_configure,
1381                                 -ENOTSUP);
1382
1383         if (sess->sess_private_data[index] == NULL) {
1384                 ret = dev->dev_ops->asym_session_configure(dev,
1385                                                         xforms,
1386                                                         sess, mp);
1387                 if (ret < 0) {
1388                         CDEV_LOG_ERR(
1389                                 "dev_id %d failed to configure session details",
1390                                 dev_id);
1391                         return ret;
1392                 }
1393         }
1394
1395         rte_cryptodev_trace_asym_session_init(dev_id, sess, xforms, mp);
1396         return 0;
1397 }
1398
1399 struct rte_mempool *
1400 rte_cryptodev_sym_session_pool_create(const char *name, uint32_t nb_elts,
1401         uint32_t elt_size, uint32_t cache_size, uint16_t user_data_size,
1402         int socket_id)
1403 {
1404         struct rte_mempool *mp;
1405         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1406         uint32_t obj_sz;
1407
1408         obj_sz = rte_cryptodev_sym_get_header_session_size() + user_data_size;
1409         if (obj_sz > elt_size)
1410                 CDEV_LOG_INFO("elt_size %u is expanded to %u\n", elt_size,
1411                                 obj_sz);
1412         else
1413                 obj_sz = elt_size;
1414
1415         mp = rte_mempool_create(name, nb_elts, obj_sz, cache_size,
1416                         (uint32_t)(sizeof(*pool_priv)),
1417                         NULL, NULL, NULL, NULL,
1418                         socket_id, 0);
1419         if (mp == NULL) {
1420                 CDEV_LOG_ERR("%s(name=%s) failed, rte_errno=%d\n",
1421                         __func__, name, rte_errno);
1422                 return NULL;
1423         }
1424
1425         pool_priv = rte_mempool_get_priv(mp);
1426         if (!pool_priv) {
1427                 CDEV_LOG_ERR("%s(name=%s) failed to get private data\n",
1428                         __func__, name);
1429                 rte_mempool_free(mp);
1430                 return NULL;
1431         }
1432
1433         pool_priv->nb_drivers = nb_drivers;
1434         pool_priv->user_data_sz = user_data_size;
1435
1436         rte_cryptodev_trace_sym_session_pool_create(name, nb_elts,
1437                 elt_size, cache_size, user_data_size, mp);
1438         return mp;
1439 }
1440
1441 static unsigned int
1442 rte_cryptodev_sym_session_data_size(struct rte_cryptodev_sym_session *sess)
1443 {
1444         return (sizeof(sess->sess_data[0]) * sess->nb_drivers) +
1445                         sess->user_data_sz;
1446 }
1447
1448 static uint8_t
1449 rte_cryptodev_sym_is_valid_session_pool(struct rte_mempool *mp)
1450 {
1451         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1452
1453         if (!mp)
1454                 return 0;
1455
1456         pool_priv = rte_mempool_get_priv(mp);
1457
1458         if (!pool_priv || mp->private_data_size < sizeof(*pool_priv) ||
1459                         pool_priv->nb_drivers != nb_drivers ||
1460                         mp->elt_size <
1461                                 rte_cryptodev_sym_get_header_session_size()
1462                                 + pool_priv->user_data_sz)
1463                 return 0;
1464
1465         return 1;
1466 }
1467
1468 struct rte_cryptodev_sym_session *
1469 rte_cryptodev_sym_session_create(struct rte_mempool *mp)
1470 {
1471         struct rte_cryptodev_sym_session *sess;
1472         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1473
1474         if (!rte_cryptodev_sym_is_valid_session_pool(mp)) {
1475                 CDEV_LOG_ERR("Invalid mempool\n");
1476                 return NULL;
1477         }
1478
1479         pool_priv = rte_mempool_get_priv(mp);
1480
1481         /* Allocate a session structure from the session pool */
1482         if (rte_mempool_get(mp, (void **)&sess)) {
1483                 CDEV_LOG_ERR("couldn't get object from session mempool");
1484                 return NULL;
1485         }
1486
1487         sess->nb_drivers = pool_priv->nb_drivers;
1488         sess->user_data_sz = pool_priv->user_data_sz;
1489         sess->opaque_data = 0;
1490
1491         /* Clear device session pointer.
1492          * Include the flag indicating presence of user data
1493          */
1494         memset(sess->sess_data, 0,
1495                         rte_cryptodev_sym_session_data_size(sess));
1496
1497         rte_cryptodev_trace_sym_session_create(mp, sess);
1498         return sess;
1499 }
1500
1501 struct rte_cryptodev_asym_session *
1502 rte_cryptodev_asym_session_create(struct rte_mempool *mp)
1503 {
1504         struct rte_cryptodev_asym_session *sess;
1505         unsigned int session_size =
1506                         rte_cryptodev_asym_get_header_session_size();
1507
1508         if (!mp) {
1509                 CDEV_LOG_ERR("invalid mempool\n");
1510                 return NULL;
1511         }
1512
1513         /* Verify if provided mempool can hold elements big enough. */
1514         if (mp->elt_size < session_size) {
1515                 CDEV_LOG_ERR(
1516                         "mempool elements too small to hold session objects");
1517                 return NULL;
1518         }
1519
1520         /* Allocate a session structure from the session pool */
1521         if (rte_mempool_get(mp, (void **)&sess)) {
1522                 CDEV_LOG_ERR("couldn't get object from session mempool");
1523                 return NULL;
1524         }
1525
1526         /* Clear device session pointer.
1527          * Include the flag indicating presence of private data
1528          */
1529         memset(sess, 0, session_size);
1530
1531         rte_cryptodev_trace_asym_session_create(mp, sess);
1532         return sess;
1533 }
1534
1535 int
1536 rte_cryptodev_sym_session_clear(uint8_t dev_id,
1537                 struct rte_cryptodev_sym_session *sess)
1538 {
1539         struct rte_cryptodev *dev;
1540         uint8_t driver_id;
1541
1542         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1543                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1544                 return -EINVAL;
1545         }
1546
1547         dev = rte_cryptodev_pmd_get_dev(dev_id);
1548
1549         if (dev == NULL || sess == NULL)
1550                 return -EINVAL;
1551
1552         driver_id = dev->driver_id;
1553         if (sess->sess_data[driver_id].refcnt == 0)
1554                 return 0;
1555         if (--sess->sess_data[driver_id].refcnt != 0)
1556                 return -EBUSY;
1557
1558         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_clear, -ENOTSUP);
1559
1560         dev->dev_ops->sym_session_clear(dev, sess);
1561
1562         rte_cryptodev_trace_sym_session_clear(dev_id, sess);
1563         return 0;
1564 }
1565
1566 int
1567 rte_cryptodev_asym_session_clear(uint8_t dev_id,
1568                 struct rte_cryptodev_asym_session *sess)
1569 {
1570         struct rte_cryptodev *dev;
1571
1572         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1573                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1574                 return -EINVAL;
1575         }
1576
1577         dev = rte_cryptodev_pmd_get_dev(dev_id);
1578
1579         if (dev == NULL || sess == NULL)
1580                 return -EINVAL;
1581
1582         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_clear, -ENOTSUP);
1583
1584         dev->dev_ops->asym_session_clear(dev, sess);
1585
1586         rte_cryptodev_trace_sym_session_clear(dev_id, sess);
1587         return 0;
1588 }
1589
1590 int
1591 rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess)
1592 {
1593         uint8_t i;
1594         struct rte_mempool *sess_mp;
1595
1596         if (sess == NULL)
1597                 return -EINVAL;
1598
1599         /* Check that all device private data has been freed */
1600         for (i = 0; i < sess->nb_drivers; i++) {
1601                 if (sess->sess_data[i].refcnt != 0)
1602                         return -EBUSY;
1603         }
1604
1605         /* Return session to mempool */
1606         sess_mp = rte_mempool_from_obj(sess);
1607         rte_mempool_put(sess_mp, sess);
1608
1609         rte_cryptodev_trace_sym_session_free(sess);
1610         return 0;
1611 }
1612
1613 int
1614 rte_cryptodev_asym_session_free(struct rte_cryptodev_asym_session *sess)
1615 {
1616         uint8_t i;
1617         void *sess_priv;
1618         struct rte_mempool *sess_mp;
1619
1620         if (sess == NULL)
1621                 return -EINVAL;
1622
1623         /* Check that all device private data has been freed */
1624         for (i = 0; i < nb_drivers; i++) {
1625                 sess_priv = get_asym_session_private_data(sess, i);
1626                 if (sess_priv != NULL)
1627                         return -EBUSY;
1628         }
1629
1630         /* Return session to mempool */
1631         sess_mp = rte_mempool_from_obj(sess);
1632         rte_mempool_put(sess_mp, sess);
1633
1634         rte_cryptodev_trace_asym_session_free(sess);
1635         return 0;
1636 }
1637
1638 unsigned int
1639 rte_cryptodev_sym_get_header_session_size(void)
1640 {
1641         /*
1642          * Header contains pointers to the private data of all registered
1643          * drivers and all necessary information to ensure safely clear
1644          * or free al session.
1645          */
1646         struct rte_cryptodev_sym_session s = {0};
1647
1648         s.nb_drivers = nb_drivers;
1649
1650         return (unsigned int)(sizeof(s) +
1651                         rte_cryptodev_sym_session_data_size(&s));
1652 }
1653
1654 unsigned int
1655 rte_cryptodev_sym_get_existing_header_session_size(
1656                 struct rte_cryptodev_sym_session *sess)
1657 {
1658         if (!sess)
1659                 return 0;
1660         else
1661                 return (unsigned int)(sizeof(*sess) +
1662                                 rte_cryptodev_sym_session_data_size(sess));
1663 }
1664
1665 unsigned int
1666 rte_cryptodev_asym_get_header_session_size(void)
1667 {
1668         /*
1669          * Header contains pointers to the private data
1670          * of all registered drivers, and a flag which
1671          * indicates presence of private data
1672          */
1673         return ((sizeof(void *) * nb_drivers) + sizeof(uint8_t));
1674 }
1675
1676 unsigned int
1677 rte_cryptodev_sym_get_private_session_size(uint8_t dev_id)
1678 {
1679         struct rte_cryptodev *dev;
1680         unsigned int priv_sess_size;
1681
1682         if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
1683                 return 0;
1684
1685         dev = rte_cryptodev_pmd_get_dev(dev_id);
1686
1687         if (*dev->dev_ops->sym_session_get_size == NULL)
1688                 return 0;
1689
1690         priv_sess_size = (*dev->dev_ops->sym_session_get_size)(dev);
1691
1692         return priv_sess_size;
1693 }
1694
1695 unsigned int
1696 rte_cryptodev_asym_get_private_session_size(uint8_t dev_id)
1697 {
1698         struct rte_cryptodev *dev;
1699         unsigned int header_size = sizeof(void *) * nb_drivers;
1700         unsigned int priv_sess_size;
1701
1702         if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
1703                 return 0;
1704
1705         dev = rte_cryptodev_pmd_get_dev(dev_id);
1706
1707         if (*dev->dev_ops->asym_session_get_size == NULL)
1708                 return 0;
1709
1710         priv_sess_size = (*dev->dev_ops->asym_session_get_size)(dev);
1711         if (priv_sess_size < header_size)
1712                 return header_size;
1713
1714         return priv_sess_size;
1715
1716 }
1717
1718 int
1719 rte_cryptodev_sym_session_set_user_data(
1720                                         struct rte_cryptodev_sym_session *sess,
1721                                         void *data,
1722                                         uint16_t size)
1723 {
1724         if (sess == NULL)
1725                 return -EINVAL;
1726
1727         if (sess->user_data_sz < size)
1728                 return -ENOMEM;
1729
1730         rte_memcpy(sess->sess_data + sess->nb_drivers, data, size);
1731         return 0;
1732 }
1733
1734 void *
1735 rte_cryptodev_sym_session_get_user_data(
1736                                         struct rte_cryptodev_sym_session *sess)
1737 {
1738         if (sess == NULL || sess->user_data_sz == 0)
1739                 return NULL;
1740
1741         return (void *)(sess->sess_data + sess->nb_drivers);
1742 }
1743
1744 static inline void
1745 sym_crypto_fill_status(struct rte_crypto_sym_vec *vec, int32_t errnum)
1746 {
1747         uint32_t i;
1748         for (i = 0; i < vec->num; i++)
1749                 vec->status[i] = errnum;
1750 }
1751
1752 uint32_t
1753 rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
1754         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs,
1755         struct rte_crypto_sym_vec *vec)
1756 {
1757         struct rte_cryptodev *dev;
1758
1759         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1760                 sym_crypto_fill_status(vec, EINVAL);
1761                 return 0;
1762         }
1763
1764         dev = rte_cryptodev_pmd_get_dev(dev_id);
1765
1766         if (*dev->dev_ops->sym_cpu_process == NULL ||
1767                 !(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO)) {
1768                 sym_crypto_fill_status(vec, ENOTSUP);
1769                 return 0;
1770         }
1771
1772         return dev->dev_ops->sym_cpu_process(dev, sess, ofs, vec);
1773 }
1774
1775 int
1776 rte_cryptodev_get_raw_dp_ctx_size(uint8_t dev_id)
1777 {
1778         struct rte_cryptodev *dev;
1779         int32_t size = sizeof(struct rte_crypto_raw_dp_ctx);
1780         int32_t priv_size;
1781
1782         if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
1783                 return -EINVAL;
1784
1785         dev = rte_cryptodev_pmd_get_dev(dev_id);
1786
1787         if (*dev->dev_ops->sym_get_raw_dp_ctx_size == NULL ||
1788                 !(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)) {
1789                 return -ENOTSUP;
1790         }
1791
1792         priv_size = (*dev->dev_ops->sym_get_raw_dp_ctx_size)(dev);
1793         if (priv_size < 0)
1794                 return -ENOTSUP;
1795
1796         return RTE_ALIGN_CEIL((size + priv_size), 8);
1797 }
1798
1799 int
1800 rte_cryptodev_configure_raw_dp_ctx(uint8_t dev_id, uint16_t qp_id,
1801         struct rte_crypto_raw_dp_ctx *ctx,
1802         enum rte_crypto_op_sess_type sess_type,
1803         union rte_cryptodev_session_ctx session_ctx,
1804         uint8_t is_update)
1805 {
1806         struct rte_cryptodev *dev;
1807
1808         if (!rte_cryptodev_get_qp_status(dev_id, qp_id))
1809                 return -EINVAL;
1810
1811         dev = rte_cryptodev_pmd_get_dev(dev_id);
1812         if (!(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)
1813                         || dev->dev_ops->sym_configure_raw_dp_ctx == NULL)
1814                 return -ENOTSUP;
1815
1816         return (*dev->dev_ops->sym_configure_raw_dp_ctx)(dev, qp_id, ctx,
1817                         sess_type, session_ctx, is_update);
1818 }
1819
1820 uint32_t
1821 rte_cryptodev_raw_enqueue_burst(struct rte_crypto_raw_dp_ctx *ctx,
1822         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
1823         void **user_data, int *enqueue_status)
1824 {
1825         return (*ctx->enqueue_burst)(ctx->qp_data, ctx->drv_ctx_data, vec,
1826                         ofs, user_data, enqueue_status);
1827 }
1828
1829 int
1830 rte_cryptodev_raw_enqueue_done(struct rte_crypto_raw_dp_ctx *ctx,
1831                 uint32_t n)
1832 {
1833         return (*ctx->enqueue_done)(ctx->qp_data, ctx->drv_ctx_data, n);
1834 }
1835
1836 uint32_t
1837 rte_cryptodev_raw_dequeue_burst(struct rte_crypto_raw_dp_ctx *ctx,
1838         rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
1839         rte_cryptodev_raw_post_dequeue_t post_dequeue,
1840         void **out_user_data, uint8_t is_user_data_array,
1841         uint32_t *n_success_jobs, int *status)
1842 {
1843         return (*ctx->dequeue_burst)(ctx->qp_data, ctx->drv_ctx_data,
1844                 get_dequeue_count, post_dequeue, out_user_data,
1845                 is_user_data_array, n_success_jobs, status);
1846 }
1847
1848 int
1849 rte_cryptodev_raw_dequeue_done(struct rte_crypto_raw_dp_ctx *ctx,
1850                 uint32_t n)
1851 {
1852         return (*ctx->dequeue_done)(ctx->qp_data, ctx->drv_ctx_data, n);
1853 }
1854
1855 /** Initialise rte_crypto_op mempool element */
1856 static void
1857 rte_crypto_op_init(struct rte_mempool *mempool,
1858                 void *opaque_arg,
1859                 void *_op_data,
1860                 __rte_unused unsigned i)
1861 {
1862         struct rte_crypto_op *op = _op_data;
1863         enum rte_crypto_op_type type = *(enum rte_crypto_op_type *)opaque_arg;
1864
1865         memset(_op_data, 0, mempool->elt_size);
1866
1867         __rte_crypto_op_reset(op, type);
1868
1869         op->phys_addr = rte_mem_virt2iova(_op_data);
1870         op->mempool = mempool;
1871 }
1872
1873
1874 struct rte_mempool *
1875 rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type,
1876                 unsigned nb_elts, unsigned cache_size, uint16_t priv_size,
1877                 int socket_id)
1878 {
1879         struct rte_crypto_op_pool_private *priv;
1880
1881         unsigned elt_size = sizeof(struct rte_crypto_op) +
1882                         priv_size;
1883
1884         if (type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
1885                 elt_size += sizeof(struct rte_crypto_sym_op);
1886         } else if (type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
1887                 elt_size += sizeof(struct rte_crypto_asym_op);
1888         } else if (type == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1889                 elt_size += RTE_MAX(sizeof(struct rte_crypto_sym_op),
1890                                     sizeof(struct rte_crypto_asym_op));
1891         } else {
1892                 CDEV_LOG_ERR("Invalid op_type\n");
1893                 return NULL;
1894         }
1895
1896         /* lookup mempool in case already allocated */
1897         struct rte_mempool *mp = rte_mempool_lookup(name);
1898
1899         if (mp != NULL) {
1900                 priv = (struct rte_crypto_op_pool_private *)
1901                                 rte_mempool_get_priv(mp);
1902
1903                 if (mp->elt_size != elt_size ||
1904                                 mp->cache_size < cache_size ||
1905                                 mp->size < nb_elts ||
1906                                 priv->priv_size <  priv_size) {
1907                         mp = NULL;
1908                         CDEV_LOG_ERR("Mempool %s already exists but with "
1909                                         "incompatible parameters", name);
1910                         return NULL;
1911                 }
1912                 return mp;
1913         }
1914
1915         mp = rte_mempool_create(
1916                         name,
1917                         nb_elts,
1918                         elt_size,
1919                         cache_size,
1920                         sizeof(struct rte_crypto_op_pool_private),
1921                         NULL,
1922                         NULL,
1923                         rte_crypto_op_init,
1924                         &type,
1925                         socket_id,
1926                         0);
1927
1928         if (mp == NULL) {
1929                 CDEV_LOG_ERR("Failed to create mempool %s", name);
1930                 return NULL;
1931         }
1932
1933         priv = (struct rte_crypto_op_pool_private *)
1934                         rte_mempool_get_priv(mp);
1935
1936         priv->priv_size = priv_size;
1937         priv->type = type;
1938
1939         return mp;
1940 }
1941
1942 int
1943 rte_cryptodev_pmd_create_dev_name(char *name, const char *dev_name_prefix)
1944 {
1945         struct rte_cryptodev *dev = NULL;
1946         uint32_t i = 0;
1947
1948         if (name == NULL)
1949                 return -EINVAL;
1950
1951         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
1952                 int ret = snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
1953                                 "%s_%u", dev_name_prefix, i);
1954
1955                 if (ret < 0)
1956                         return ret;
1957
1958                 dev = rte_cryptodev_pmd_get_named_dev(name);
1959                 if (!dev)
1960                         return 0;
1961         }
1962
1963         return -1;
1964 }
1965
1966 TAILQ_HEAD(cryptodev_driver_list, cryptodev_driver);
1967
1968 static struct cryptodev_driver_list cryptodev_driver_list =
1969         TAILQ_HEAD_INITIALIZER(cryptodev_driver_list);
1970
1971 int
1972 rte_cryptodev_driver_id_get(const char *name)
1973 {
1974         struct cryptodev_driver *driver;
1975         const char *driver_name;
1976
1977         if (name == NULL) {
1978                 RTE_LOG(DEBUG, CRYPTODEV, "name pointer NULL");
1979                 return -1;
1980         }
1981
1982         TAILQ_FOREACH(driver, &cryptodev_driver_list, next) {
1983                 driver_name = driver->driver->name;
1984                 if (strncmp(driver_name, name, strlen(driver_name) + 1) == 0)
1985                         return driver->id;
1986         }
1987         return -1;
1988 }
1989
1990 const char *
1991 rte_cryptodev_name_get(uint8_t dev_id)
1992 {
1993         struct rte_cryptodev *dev;
1994
1995         if (!rte_cryptodev_is_valid_device_data(dev_id)) {
1996                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1997                 return NULL;
1998         }
1999
2000         dev = rte_cryptodev_pmd_get_dev(dev_id);
2001         if (dev == NULL)
2002                 return NULL;
2003
2004         return dev->data->name;
2005 }
2006
2007 const char *
2008 rte_cryptodev_driver_name_get(uint8_t driver_id)
2009 {
2010         struct cryptodev_driver *driver;
2011
2012         TAILQ_FOREACH(driver, &cryptodev_driver_list, next)
2013                 if (driver->id == driver_id)
2014                         return driver->driver->name;
2015         return NULL;
2016 }
2017
2018 uint8_t
2019 rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv,
2020                 const struct rte_driver *drv)
2021 {
2022         crypto_drv->driver = drv;
2023         crypto_drv->id = nb_drivers;
2024
2025         TAILQ_INSERT_TAIL(&cryptodev_driver_list, crypto_drv, next);
2026
2027         return nb_drivers++;
2028 }