lib/cryptodev/rte_cryptodev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2020 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdint.h>
  14 #include <inttypes.h>
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_log.h>
  18 #include <rte_debug.h>
  19 #include <rte_dev.h>
  20 #include <rte_interrupts.h>
  21 #include <rte_memory.h>
  22 #include <rte_memcpy.h>
  23 #include <rte_memzone.h>
  24 #include <rte_launch.h>
  25 #include <rte_tailq.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_common.h>
  32 #include <rte_mempool.h>
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_errno.h>
  36 #include <rte_spinlock.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_telemetry.h>
  39
  40 #include "rte_crypto.h"
  41 #include "rte_cryptodev.h"
  42 #include "cryptodev_pmd.h"
  43 #include "rte_cryptodev_trace.h"
  44
  45 static uint8_t nb_drivers;
  46
  47 static struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
  48
  49 struct rte_cryptodev *rte_cryptodevs = rte_crypto_devices;
  50
  51 static struct rte_cryptodev_global cryptodev_globals = {
  52                 .devs                   = rte_crypto_devices,
  53                 .data                   = { NULL },
  54                 .nb_devs                = 0
  55 };
  56
  57 /* Public fastpath APIs. */
  58 struct rte_crypto_fp_ops rte_crypto_fp_ops[RTE_CRYPTO_MAX_DEVS];
  59
  60 /* spinlock for crypto device callbacks */
  61 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  62
  63 /**
  64  * The user application callback description.
  65  *
  66  * It contains callback address to be registered by user application,
  67  * the pointer to the parameters for callback, and the event type.
  68  */
  69 struct rte_cryptodev_callback {
  70         TAILQ_ENTRY(rte_cryptodev_callback) next; /**< Callbacks list */
  71         rte_cryptodev_cb_fn cb_fn;              /**< Callback address */
  72         void *cb_arg;                           /**< Parameter for callback */
  73         enum rte_cryptodev_event_type event;    /**< Interrupt event type */
  74         uint32_t active;                        /**< Callback is executing */
  75 };
  76
  77 /**
  78  * The crypto cipher algorithm strings identifiers.
  79  * It could be used in application command line.
  80  */
  81 const char *
  82 rte_crypto_cipher_algorithm_strings[] = {
  83         [RTE_CRYPTO_CIPHER_3DES_CBC]    = "3des-cbc",
  84         [RTE_CRYPTO_CIPHER_3DES_ECB]    = "3des-ecb",
  85         [RTE_CRYPTO_CIPHER_3DES_CTR]    = "3des-ctr",
  86
  87         [RTE_CRYPTO_CIPHER_AES_CBC]     = "aes-cbc",
  88         [RTE_CRYPTO_CIPHER_AES_CTR]     = "aes-ctr",
  89         [RTE_CRYPTO_CIPHER_AES_DOCSISBPI]       = "aes-docsisbpi",
  90         [RTE_CRYPTO_CIPHER_AES_ECB]     = "aes-ecb",
  91         [RTE_CRYPTO_CIPHER_AES_F8]      = "aes-f8",
  92         [RTE_CRYPTO_CIPHER_AES_XTS]     = "aes-xts",
  93
  94         [RTE_CRYPTO_CIPHER_ARC4]        = "arc4",
  95
  96         [RTE_CRYPTO_CIPHER_DES_CBC]     = "des-cbc",
  97         [RTE_CRYPTO_CIPHER_DES_DOCSISBPI]       = "des-docsisbpi",
  98
  99         [RTE_CRYPTO_CIPHER_NULL]        = "null",
 100
 101         [RTE_CRYPTO_CIPHER_KASUMI_F8]   = "kasumi-f8",
 102         [RTE_CRYPTO_CIPHER_SNOW3G_UEA2] = "snow3g-uea2",
 103         [RTE_CRYPTO_CIPHER_ZUC_EEA3]    = "zuc-eea3"
 104 };
 105
 106 /**
 107  * The crypto cipher operation strings identifiers.
 108  * It could be used in application command line.
 109  */
 110 const char *
 111 rte_crypto_cipher_operation_strings[] = {
 112                 [RTE_CRYPTO_CIPHER_OP_ENCRYPT]  = "encrypt",
 113                 [RTE_CRYPTO_CIPHER_OP_DECRYPT]  = "decrypt"
 114 };
 115
 116 /**
 117  * The crypto auth algorithm strings identifiers.
 118  * It could be used in application command line.
 119  */
 120 const char *
 121 rte_crypto_auth_algorithm_strings[] = {
 122         [RTE_CRYPTO_AUTH_AES_CBC_MAC]   = "aes-cbc-mac",
 123         [RTE_CRYPTO_AUTH_AES_CMAC]      = "aes-cmac",
 124         [RTE_CRYPTO_AUTH_AES_GMAC]      = "aes-gmac",
 125         [RTE_CRYPTO_AUTH_AES_XCBC_MAC]  = "aes-xcbc-mac",
 126
 127         [RTE_CRYPTO_AUTH_MD5]           = "md5",
 128         [RTE_CRYPTO_AUTH_MD5_HMAC]      = "md5-hmac",
 129
 130         [RTE_CRYPTO_AUTH_NULL]          = "null",
 131
 132         [RTE_CRYPTO_AUTH_SHA1]          = "sha1",
 133         [RTE_CRYPTO_AUTH_SHA1_HMAC]     = "sha1-hmac",
 134
 135         [RTE_CRYPTO_AUTH_SHA224]        = "sha2-224",
 136         [RTE_CRYPTO_AUTH_SHA224_HMAC]   = "sha2-224-hmac",
 137         [RTE_CRYPTO_AUTH_SHA256]        = "sha2-256",
 138         [RTE_CRYPTO_AUTH_SHA256_HMAC]   = "sha2-256-hmac",
 139         [RTE_CRYPTO_AUTH_SHA384]        = "sha2-384",
 140         [RTE_CRYPTO_AUTH_SHA384_HMAC]   = "sha2-384-hmac",
 141         [RTE_CRYPTO_AUTH_SHA512]        = "sha2-512",
 142         [RTE_CRYPTO_AUTH_SHA512_HMAC]   = "sha2-512-hmac",
 143
 144         [RTE_CRYPTO_AUTH_KASUMI_F9]     = "kasumi-f9",
 145         [RTE_CRYPTO_AUTH_SNOW3G_UIA2]   = "snow3g-uia2",
 146         [RTE_CRYPTO_AUTH_ZUC_EIA3]      = "zuc-eia3"
 147 };
 148
 149 /**
 150  * The crypto AEAD algorithm strings identifiers.
 151  * It could be used in application command line.
 152  */
 153 const char *
 154 rte_crypto_aead_algorithm_strings[] = {
 155         [RTE_CRYPTO_AEAD_AES_CCM]       = "aes-ccm",
 156         [RTE_CRYPTO_AEAD_AES_GCM]       = "aes-gcm",
 157         [RTE_CRYPTO_AEAD_CHACHA20_POLY1305] = "chacha20-poly1305"
 158 };
 159
 160 /**
 161  * The crypto AEAD operation strings identifiers.
 162  * It could be used in application command line.
 163  */
 164 const char *
 165 rte_crypto_aead_operation_strings[] = {
 166         [RTE_CRYPTO_AEAD_OP_ENCRYPT]    = "encrypt",
 167         [RTE_CRYPTO_AEAD_OP_DECRYPT]    = "decrypt"
 168 };
 169
 170 /**
 171  * Asymmetric crypto transform operation strings identifiers.
 172  */
 173 const char *rte_crypto_asym_xform_strings[] = {
 174         [RTE_CRYPTO_ASYM_XFORM_NONE]    = "none",
 175         [RTE_CRYPTO_ASYM_XFORM_RSA]     = "rsa",
 176         [RTE_CRYPTO_ASYM_XFORM_MODEX]   = "modexp",
 177         [RTE_CRYPTO_ASYM_XFORM_MODINV]  = "modinv",
 178         [RTE_CRYPTO_ASYM_XFORM_DH]      = "dh",
 179         [RTE_CRYPTO_ASYM_XFORM_DSA]     = "dsa",
 180         [RTE_CRYPTO_ASYM_XFORM_ECDSA]   = "ecdsa",
 181         [RTE_CRYPTO_ASYM_XFORM_ECPM]    = "ecpm",
 182 };
 183
 184 /**
 185  * Asymmetric crypto operation strings identifiers.
 186  */
 187 const char *rte_crypto_asym_op_strings[] = {
 188         [RTE_CRYPTO_ASYM_OP_ENCRYPT]    = "encrypt",
 189         [RTE_CRYPTO_ASYM_OP_DECRYPT]    = "decrypt",
 190         [RTE_CRYPTO_ASYM_OP_SIGN]       = "sign",
 191         [RTE_CRYPTO_ASYM_OP_VERIFY]     = "verify",
 192         [RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE]       = "priv_key_generate",
 193         [RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE] = "pub_key_generate",
 194         [RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE] = "sharedsecret_compute",
 195 };
 196
 197 /**
 198  * The private data structure stored in the sym session mempool private data.
 199  */
 200 struct rte_cryptodev_sym_session_pool_private_data {
 201         uint16_t nb_drivers;
 202         /**< number of elements in sess_data array */
 203         uint16_t user_data_sz;
 204         /**< session user data will be placed after sess_data */
 205 };
 206
 207 /**
 208  * The private data structure stored in the asym session mempool private data.
 209  */
 210 struct rte_cryptodev_asym_session_pool_private_data {
 211         uint16_t max_priv_session_sz;
 212         /**< Size of private session data used when creating mempool */
 213 };
 214
 215 int
 216 rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum,
 217                 const char *algo_string)
 218 {
 219         unsigned int i;
 220
 221         for (i = 1; i < RTE_DIM(rte_crypto_cipher_algorithm_strings); i++) {
 222                 if (strcmp(algo_string, rte_crypto_cipher_algorithm_strings[i]) == 0) {
 223                         *algo_enum = (enum rte_crypto_cipher_algorithm) i;
 224                         return 0;
 225                 }
 226         }
 227
 228         /* Invalid string */
 229         return -1;
 230 }
 231
 232 int
 233 rte_cryptodev_get_auth_algo_enum(enum rte_crypto_auth_algorithm *algo_enum,
 234                 const char *algo_string)
 235 {
 236         unsigned int i;
 237
 238         for (i = 1; i < RTE_DIM(rte_crypto_auth_algorithm_strings); i++) {
 239                 if (strcmp(algo_string, rte_crypto_auth_algorithm_strings[i]) == 0) {
 240                         *algo_enum = (enum rte_crypto_auth_algorithm) i;
 241                         return 0;
 242                 }
 243         }
 244
 245         /* Invalid string */
 246         return -1;
 247 }
 248
 249 int
 250 rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum,
 251                 const char *algo_string)
 252 {
 253         unsigned int i;
 254
 255         for (i = 1; i < RTE_DIM(rte_crypto_aead_algorithm_strings); i++) {
 256                 if (strcmp(algo_string, rte_crypto_aead_algorithm_strings[i]) == 0) {
 257                         *algo_enum = (enum rte_crypto_aead_algorithm) i;
 258                         return 0;
 259                 }
 260         }
 261
 262         /* Invalid string */
 263         return -1;
 264 }
 265
 266 int
 267 rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
 268                 const char *xform_string)
 269 {
 270         unsigned int i;
 271
 272         for (i = 1; i < RTE_DIM(rte_crypto_asym_xform_strings); i++) {
 273                 if (strcmp(xform_string,
 274                         rte_crypto_asym_xform_strings[i]) == 0) {
 275                         *xform_enum = (enum rte_crypto_asym_xform_type) i;
 276                         return 0;
 277                 }
 278         }
 279
 280         /* Invalid string */
 281         return -1;
 282 }
 283
 284 /**
 285  * The crypto auth operation strings identifiers.
 286  * It could be used in application command line.
 287  */
 288 const char *
 289 rte_crypto_auth_operation_strings[] = {
 290                 [RTE_CRYPTO_AUTH_OP_VERIFY]     = "verify",
 291                 [RTE_CRYPTO_AUTH_OP_GENERATE]   = "generate"
 292 };
 293
 294 const struct rte_cryptodev_symmetric_capability *
 295 rte_cryptodev_sym_capability_get(uint8_t dev_id,
 296                 const struct rte_cryptodev_sym_capability_idx *idx)
 297 {
 298         const struct rte_cryptodev_capabilities *capability;
 299         struct rte_cryptodev_info dev_info;
 300         int i = 0;
 301
 302         rte_cryptodev_info_get(dev_id, &dev_info);
 303
 304         while ((capability = &dev_info.capabilities[i++])->op !=
 305                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 306                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
 307                         continue;
 308
 309                 if (capability->sym.xform_type != idx->type)
 310                         continue;
 311
 312                 if (idx->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 313                         capability->sym.auth.algo == idx->algo.auth)
 314                         return &capability->sym;
 315
 316                 if (idx->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 317                         capability->sym.cipher.algo == idx->algo.cipher)
 318                         return &capability->sym;
 319
 320                 if (idx->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
 321                                 capability->sym.aead.algo == idx->algo.aead)
 322                         return &capability->sym;
 323         }
 324
 325         return NULL;
 326 }
 327
 328 static int
 329 param_range_check(uint16_t size, const struct rte_crypto_param_range *range)
 330 {
 331         unsigned int next_size;
 332
 333         /* Check lower/upper bounds */
 334         if (size < range->min)
 335                 return -1;
 336
 337         if (size > range->max)
 338                 return -1;
 339
 340         /* If range is actually only one value, size is correct */
 341         if (range->increment == 0)
 342                 return 0;
 343
 344         /* Check if value is one of the supported sizes */
 345         for (next_size = range->min; next_size <= range->max;
 346                         next_size += range->increment)
 347                 if (size == next_size)
 348                         return 0;
 349
 350         return -1;
 351 }
 352
 353 const struct rte_cryptodev_asymmetric_xform_capability *
 354 rte_cryptodev_asym_capability_get(uint8_t dev_id,
 355                 const struct rte_cryptodev_asym_capability_idx *idx)
 356 {
 357         const struct rte_cryptodev_capabilities *capability;
 358         struct rte_cryptodev_info dev_info;
 359         unsigned int i = 0;
 360
 361         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 362         rte_cryptodev_info_get(dev_id, &dev_info);
 363
 364         while ((capability = &dev_info.capabilities[i++])->op !=
 365                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 366                 if (capability->op != RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
 367                         continue;
 368
 369                 if (capability->asym.xform_capa.xform_type == idx->type)
 370                         return &capability->asym.xform_capa;
 371         }
 372         return NULL;
 373 };
 374
 375 int
 376 rte_cryptodev_sym_capability_check_cipher(
 377                 const struct rte_cryptodev_symmetric_capability *capability,
 378                 uint16_t key_size, uint16_t iv_size)
 379 {
 380         if (param_range_check(key_size, &capability->cipher.key_size) != 0)
 381                 return -1;
 382
 383         if (param_range_check(iv_size, &capability->cipher.iv_size) != 0)
 384                 return -1;
 385
 386         return 0;
 387 }
 388
 389 int
 390 rte_cryptodev_sym_capability_check_auth(
 391                 const struct rte_cryptodev_symmetric_capability *capability,
 392                 uint16_t key_size, uint16_t digest_size, uint16_t iv_size)
 393 {
 394         if (param_range_check(key_size, &capability->auth.key_size) != 0)
 395                 return -1;
 396
 397         if (param_range_check(digest_size, &capability->auth.digest_size) != 0)
 398                 return -1;
 399
 400         if (param_range_check(iv_size, &capability->auth.iv_size) != 0)
 401                 return -1;
 402
 403         return 0;
 404 }
 405
 406 int
 407 rte_cryptodev_sym_capability_check_aead(
 408                 const struct rte_cryptodev_symmetric_capability *capability,
 409                 uint16_t key_size, uint16_t digest_size, uint16_t aad_size,
 410                 uint16_t iv_size)
 411 {
 412         if (param_range_check(key_size, &capability->aead.key_size) != 0)
 413                 return -1;
 414
 415         if (param_range_check(digest_size, &capability->aead.digest_size) != 0)
 416                 return -1;
 417
 418         if (param_range_check(aad_size, &capability->aead.aad_size) != 0)
 419                 return -1;
 420
 421         if (param_range_check(iv_size, &capability->aead.iv_size) != 0)
 422                 return -1;
 423
 424         return 0;
 425 }
 426 int
 427 rte_cryptodev_asym_xform_capability_check_optype(
 428         const struct rte_cryptodev_asymmetric_xform_capability *capability,
 429         enum rte_crypto_asym_op_type op_type)
 430 {
 431         if (capability->op_types & (1 << op_type))
 432                 return 1;
 433
 434         return 0;
 435 }
 436
 437 int
 438 rte_cryptodev_asym_xform_capability_check_modlen(
 439         const struct rte_cryptodev_asymmetric_xform_capability *capability,
 440         uint16_t modlen)
 441 {
 442         /* no need to check for limits, if min or max = 0 */
 443         if (capability->modlen.min != 0) {
 444                 if (modlen < capability->modlen.min)
 445                         return -1;
 446         }
 447
 448         if (capability->modlen.max != 0) {
 449                 if (modlen > capability->modlen.max)
 450                         return -1;
 451         }
 452
 453         /* in any case, check if given modlen is module increment */
 454         if (capability->modlen.increment != 0) {
 455                 if (modlen % (capability->modlen.increment))
 456                         return -1;
 457         }
 458
 459         return 0;
 460 }
 461
 462 /* spinlock for crypto device enq callbacks */
 463 static rte_spinlock_t rte_cryptodev_callback_lock = RTE_SPINLOCK_INITIALIZER;
 464
 465 static void
 466 cryptodev_cb_cleanup(struct rte_cryptodev *dev)
 467 {
 468         struct rte_cryptodev_cb_rcu *list;
 469         struct rte_cryptodev_cb *cb, *next;
 470         uint16_t qp_id;
 471
 472         if (dev->enq_cbs == NULL && dev->deq_cbs == NULL)
 473                 return;
 474
 475         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 476                 list = &dev->enq_cbs[qp_id];
 477                 cb = list->next;
 478                 while (cb != NULL) {
 479                         next = cb->next;
 480                         rte_free(cb);
 481                         cb = next;
 482                 }
 483
 484                 rte_free(list->qsbr);
 485         }
 486
 487         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 488                 list = &dev->deq_cbs[qp_id];
 489                 cb = list->next;
 490                 while (cb != NULL) {
 491                         next = cb->next;
 492                         rte_free(cb);
 493                         cb = next;
 494                 }
 495
 496                 rte_free(list->qsbr);
 497         }
 498
 499         rte_free(dev->enq_cbs);
 500         dev->enq_cbs = NULL;
 501         rte_free(dev->deq_cbs);
 502         dev->deq_cbs = NULL;
 503 }
 504
 505 static int
 506 cryptodev_cb_init(struct rte_cryptodev *dev)
 507 {
 508         struct rte_cryptodev_cb_rcu *list;
 509         struct rte_rcu_qsbr *qsbr;
 510         uint16_t qp_id;
 511         size_t size;
 512
 513         /* Max thread set to 1, as one DP thread accessing a queue-pair */
 514         const uint32_t max_threads = 1;
 515
 516         dev->enq_cbs = rte_zmalloc(NULL,
 517                                    sizeof(struct rte_cryptodev_cb_rcu) *
 518                                    dev->data->nb_queue_pairs, 0);
 519         if (dev->enq_cbs == NULL) {
 520                 CDEV_LOG_ERR("Failed to allocate memory for enq callbacks");
 521                 return -ENOMEM;
 522         }
 523
 524         dev->deq_cbs = rte_zmalloc(NULL,
 525                                    sizeof(struct rte_cryptodev_cb_rcu) *
 526                                    dev->data->nb_queue_pairs, 0);
 527         if (dev->deq_cbs == NULL) {
 528                 CDEV_LOG_ERR("Failed to allocate memory for deq callbacks");
 529                 rte_free(dev->enq_cbs);
 530                 return -ENOMEM;
 531         }
 532
 533         /* Create RCU QSBR variable */
 534         size = rte_rcu_qsbr_get_memsize(max_threads);
 535
 536         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 537                 list = &dev->enq_cbs[qp_id];
 538                 qsbr = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 539                 if (qsbr == NULL) {
 540                         CDEV_LOG_ERR("Failed to allocate memory for RCU on "
 541                                 "queue_pair_id=%d", qp_id);
 542                         goto cb_init_err;
 543                 }
 544
 545                 if (rte_rcu_qsbr_init(qsbr, max_threads)) {
 546                         CDEV_LOG_ERR("Failed to initialize for RCU on "
 547                                 "queue_pair_id=%d", qp_id);
 548                         goto cb_init_err;
 549                 }
 550
 551                 list->qsbr = qsbr;
 552         }
 553
 554         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 555                 list = &dev->deq_cbs[qp_id];
 556                 qsbr = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 557                 if (qsbr == NULL) {
 558                         CDEV_LOG_ERR("Failed to allocate memory for RCU on "
 559                                 "queue_pair_id=%d", qp_id);
 560                         goto cb_init_err;
 561                 }
 562
 563                 if (rte_rcu_qsbr_init(qsbr, max_threads)) {
 564                         CDEV_LOG_ERR("Failed to initialize for RCU on "
 565                                 "queue_pair_id=%d", qp_id);
 566                         goto cb_init_err;
 567                 }
 568
 569                 list->qsbr = qsbr;
 570         }
 571
 572         return 0;
 573
 574 cb_init_err:
 575         cryptodev_cb_cleanup(dev);
 576         return -ENOMEM;
 577 }
 578
 579 const char *
 580 rte_cryptodev_get_feature_name(uint64_t flag)
 581 {
 582         switch (flag) {
 583         case RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO:
 584                 return "SYMMETRIC_CRYPTO";
 585         case RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO:
 586                 return "ASYMMETRIC_CRYPTO";
 587         case RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING:
 588                 return "SYM_OPERATION_CHAINING";
 589         case RTE_CRYPTODEV_FF_CPU_SSE:
 590                 return "CPU_SSE";
 591         case RTE_CRYPTODEV_FF_CPU_AVX:
 592                 return "CPU_AVX";
 593         case RTE_CRYPTODEV_FF_CPU_AVX2:
 594                 return "CPU_AVX2";
 595         case RTE_CRYPTODEV_FF_CPU_AVX512:
 596                 return "CPU_AVX512";
 597         case RTE_CRYPTODEV_FF_CPU_AESNI:
 598                 return "CPU_AESNI";
 599         case RTE_CRYPTODEV_FF_HW_ACCELERATED:
 600                 return "HW_ACCELERATED";
 601         case RTE_CRYPTODEV_FF_IN_PLACE_SGL:
 602                 return "IN_PLACE_SGL";
 603         case RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT:
 604                 return "OOP_SGL_IN_SGL_OUT";
 605         case RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT:
 606                 return "OOP_SGL_IN_LB_OUT";
 607         case RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT:
 608                 return "OOP_LB_IN_SGL_OUT";
 609         case RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT:
 610                 return "OOP_LB_IN_LB_OUT";
 611         case RTE_CRYPTODEV_FF_CPU_NEON:
 612                 return "CPU_NEON";
 613         case RTE_CRYPTODEV_FF_CPU_ARM_CE:
 614                 return "CPU_ARM_CE";
 615         case RTE_CRYPTODEV_FF_SECURITY:
 616                 return "SECURITY_PROTOCOL";
 617         case RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP:
 618                 return "RSA_PRIV_OP_KEY_EXP";
 619         case RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT:
 620                 return "RSA_PRIV_OP_KEY_QT";
 621         case RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED:
 622                 return "DIGEST_ENCRYPTED";
 623         case RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO:
 624                 return "SYM_CPU_CRYPTO";
 625         case RTE_CRYPTODEV_FF_ASYM_SESSIONLESS:
 626                 return "ASYM_SESSIONLESS";
 627         case RTE_CRYPTODEV_FF_SYM_SESSIONLESS:
 628                 return "SYM_SESSIONLESS";
 629         case RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA:
 630                 return "NON_BYTE_ALIGNED_DATA";
 631         case RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS:
 632                 return "CIPHER_MULTIPLE_DATA_UNITS";
 633         case RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY:
 634                 return "CIPHER_WRAPPED_KEY";
 635         default:
 636                 return NULL;
 637         }
 638 }
 639
 640 struct rte_cryptodev *
 641 rte_cryptodev_pmd_get_dev(uint8_t dev_id)
 642 {
 643         return &cryptodev_globals.devs[dev_id];
 644 }
 645
 646 struct rte_cryptodev *
 647 rte_cryptodev_pmd_get_named_dev(const char *name)
 648 {
 649         struct rte_cryptodev *dev;
 650         unsigned int i;
 651
 652         if (name == NULL)
 653                 return NULL;
 654
 655         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
 656                 dev = &cryptodev_globals.devs[i];
 657
 658                 if ((dev->attached == RTE_CRYPTODEV_ATTACHED) &&
 659                                 (strcmp(dev->data->name, name) == 0))
 660                         return dev;
 661         }
 662
 663         return NULL;
 664 }
 665
 666 static inline uint8_t
 667 rte_cryptodev_is_valid_device_data(uint8_t dev_id)
 668 {
 669         if (dev_id >= RTE_CRYPTO_MAX_DEVS ||
 670                         rte_crypto_devices[dev_id].data == NULL)
 671                 return 0;
 672
 673         return 1;
 674 }
 675
 676 unsigned int
 677 rte_cryptodev_is_valid_dev(uint8_t dev_id)
 678 {
 679         struct rte_cryptodev *dev = NULL;
 680
 681         if (!rte_cryptodev_is_valid_device_data(dev_id))
 682                 return 0;
 683
 684         dev = rte_cryptodev_pmd_get_dev(dev_id);
 685         if (dev->attached != RTE_CRYPTODEV_ATTACHED)
 686                 return 0;
 687         else
 688                 return 1;
 689 }
 690
 691
 692 int
 693 rte_cryptodev_get_dev_id(const char *name)
 694 {
 695         unsigned i;
 696
 697         if (name == NULL)
 698                 return -1;
 699
 700         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
 701                 if (!rte_cryptodev_is_valid_device_data(i))
 702                         continue;
 703                 if ((strcmp(cryptodev_globals.devs[i].data->name, name)
 704                                 == 0) &&
 705                                 (cryptodev_globals.devs[i].attached ==
 706                                                 RTE_CRYPTODEV_ATTACHED))
 707                         return i;
 708         }
 709
 710         return -1;
 711 }
 712
 713 uint8_t
 714 rte_cryptodev_count(void)
 715 {
 716         return cryptodev_globals.nb_devs;
 717 }
 718
 719 uint8_t
 720 rte_cryptodev_device_count_by_driver(uint8_t driver_id)
 721 {
 722         uint8_t i, dev_count = 0;
 723
 724         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++)
 725                 if (cryptodev_globals.devs[i].driver_id == driver_id &&
 726                         cryptodev_globals.devs[i].attached ==
 727                                         RTE_CRYPTODEV_ATTACHED)
 728                         dev_count++;
 729
 730         return dev_count;
 731 }
 732
 733 uint8_t
 734 rte_cryptodev_devices_get(const char *driver_name, uint8_t *devices,
 735         uint8_t nb_devices)
 736 {
 737         uint8_t i, count = 0;
 738         struct rte_cryptodev *devs = cryptodev_globals.devs;
 739
 740         for (i = 0; i < RTE_CRYPTO_MAX_DEVS && count < nb_devices; i++) {
 741                 if (!rte_cryptodev_is_valid_device_data(i))
 742                         continue;
 743
 744                 if (devs[i].attached == RTE_CRYPTODEV_ATTACHED) {
 745                         int cmp;
 746
 747                         cmp = strncmp(devs[i].device->driver->name,
 748                                         driver_name,
 749                                         strlen(driver_name) + 1);
 750
 751                         if (cmp == 0)
 752                                 devices[count++] = devs[i].data->dev_id;
 753                 }
 754         }
 755
 756         return count;
 757 }
 758
 759 void *
 760 rte_cryptodev_get_sec_ctx(uint8_t dev_id)
 761 {
 762         if (dev_id < RTE_CRYPTO_MAX_DEVS &&
 763                         (rte_crypto_devices[dev_id].feature_flags &
 764                         RTE_CRYPTODEV_FF_SECURITY))
 765                 return rte_crypto_devices[dev_id].security_ctx;
 766
 767         return NULL;
 768 }
 769
 770 int
 771 rte_cryptodev_socket_id(uint8_t dev_id)
 772 {
 773         struct rte_cryptodev *dev;
 774
 775         if (!rte_cryptodev_is_valid_dev(dev_id))
 776                 return -1;
 777
 778         dev = rte_cryptodev_pmd_get_dev(dev_id);
 779
 780         return dev->data->socket_id;
 781 }
 782
 783 static inline int
 784 rte_cryptodev_data_alloc(uint8_t dev_id, struct rte_cryptodev_data **data,
 785                 int socket_id)
 786 {
 787         char mz_name[RTE_MEMZONE_NAMESIZE];
 788         const struct rte_memzone *mz;
 789         int n;
 790
 791         /* generate memzone name */
 792         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 793         if (n >= (int)sizeof(mz_name))
 794                 return -EINVAL;
 795
 796         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 797                 mz = rte_memzone_reserve(mz_name,
 798                                 sizeof(struct rte_cryptodev_data),
 799                                 socket_id, 0);
 800                 CDEV_LOG_DEBUG("PRIMARY:reserved memzone for %s (%p)",
 801                                 mz_name, mz);
 802         } else {
 803                 mz = rte_memzone_lookup(mz_name);
 804                 CDEV_LOG_DEBUG("SECONDARY:looked up memzone for %s (%p)",
 805                                 mz_name, mz);
 806         }
 807
 808         if (mz == NULL)
 809                 return -ENOMEM;
 810
 811         *data = mz->addr;
 812         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 813                 memset(*data, 0, sizeof(struct rte_cryptodev_data));
 814
 815         return 0;
 816 }
 817
 818 static inline int
 819 rte_cryptodev_data_free(uint8_t dev_id, struct rte_cryptodev_data **data)
 820 {
 821         char mz_name[RTE_MEMZONE_NAMESIZE];
 822         const struct rte_memzone *mz;
 823         int n;
 824
 825         /* generate memzone name */
 826         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 827         if (n >= (int)sizeof(mz_name))
 828                 return -EINVAL;
 829
 830         mz = rte_memzone_lookup(mz_name);
 831         if (mz == NULL)
 832                 return -ENOMEM;
 833
 834         RTE_ASSERT(*data == mz->addr);
 835         *data = NULL;
 836
 837         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 838                 CDEV_LOG_DEBUG("PRIMARY:free memzone of %s (%p)",
 839                                 mz_name, mz);
 840                 return rte_memzone_free(mz);
 841         } else {
 842                 CDEV_LOG_DEBUG("SECONDARY:don't free memzone of %s (%p)",
 843                                 mz_name, mz);
 844         }
 845
 846         return 0;
 847 }
 848
 849 static uint8_t
 850 rte_cryptodev_find_free_device_index(void)
 851 {
 852         uint8_t dev_id;
 853
 854         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++) {
 855                 if (rte_crypto_devices[dev_id].attached ==
 856                                 RTE_CRYPTODEV_DETACHED)
 857                         return dev_id;
 858         }
 859         return RTE_CRYPTO_MAX_DEVS;
 860 }
 861
 862 struct rte_cryptodev *
 863 rte_cryptodev_pmd_allocate(const char *name, int socket_id)
 864 {
 865         struct rte_cryptodev *cryptodev;
 866         uint8_t dev_id;
 867
 868         if (rte_cryptodev_pmd_get_named_dev(name) != NULL) {
 869                 CDEV_LOG_ERR("Crypto device with name %s already "
 870                                 "allocated!", name);
 871                 return NULL;
 872         }
 873
 874         dev_id = rte_cryptodev_find_free_device_index();
 875         if (dev_id == RTE_CRYPTO_MAX_DEVS) {
 876                 CDEV_LOG_ERR("Reached maximum number of crypto devices");
 877                 return NULL;
 878         }
 879
 880         cryptodev = rte_cryptodev_pmd_get_dev(dev_id);
 881
 882         if (cryptodev->data == NULL) {
 883                 struct rte_cryptodev_data **cryptodev_data =
 884                                 &cryptodev_globals.data[dev_id];
 885
 886                 int retval = rte_cryptodev_data_alloc(dev_id, cryptodev_data,
 887                                 socket_id);
 888
 889                 if (retval < 0 || *cryptodev_data == NULL)
 890                         return NULL;
 891
 892                 cryptodev->data = *cryptodev_data;
 893
 894                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 895                         strlcpy(cryptodev->data->name, name,
 896                                 RTE_CRYPTODEV_NAME_MAX_LEN);
 897
 898                         cryptodev->data->dev_id = dev_id;
 899                         cryptodev->data->socket_id = socket_id;
 900                         cryptodev->data->dev_started = 0;
 901                         CDEV_LOG_DEBUG("PRIMARY:init data");
 902                 }
 903
 904                 CDEV_LOG_DEBUG("Data for %s: dev_id %d, socket %d, started %d",
 905                                 cryptodev->data->name,
 906                                 cryptodev->data->dev_id,
 907                                 cryptodev->data->socket_id,
 908                                 cryptodev->data->dev_started);
 909
 910                 /* init user callbacks */
 911                 TAILQ_INIT(&(cryptodev->link_intr_cbs));
 912
 913                 cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 914
 915                 cryptodev_globals.nb_devs++;
 916         }
 917
 918         return cryptodev;
 919 }
 920
 921 int
 922 rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 923 {
 924         int ret;
 925         uint8_t dev_id;
 926
 927         if (cryptodev == NULL)
 928                 return -EINVAL;
 929
 930         dev_id = cryptodev->data->dev_id;
 931
 932         cryptodev_fp_ops_reset(rte_crypto_fp_ops + dev_id);
 933
 934         /* Close device only if device operations have been set */
 935         if (cryptodev->dev_ops) {
 936                 ret = rte_cryptodev_close(dev_id);
 937                 if (ret < 0)
 938                         return ret;
 939         }
 940
 941         ret = rte_cryptodev_data_free(dev_id, &cryptodev_globals.data[dev_id]);
 942         if (ret < 0)
 943                 return ret;
 944
 945         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 946         cryptodev_globals.nb_devs--;
 947         return 0;
 948 }
 949
 950 uint16_t
 951 rte_cryptodev_queue_pair_count(uint8_t dev_id)
 952 {
 953         struct rte_cryptodev *dev;
 954
 955         if (!rte_cryptodev_is_valid_device_data(dev_id)) {
 956                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 957                 return 0;
 958         }
 959
 960         dev = &rte_crypto_devices[dev_id];
 961         return dev->data->nb_queue_pairs;
 962 }
 963
 964 static int
 965 rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 966                 int socket_id)
 967 {
 968         struct rte_cryptodev_info dev_info;
 969         void **qp;
 970         unsigned i;
 971
 972         if ((dev == NULL) || (nb_qpairs < 1)) {
 973                 CDEV_LOG_ERR("invalid param: dev %p, nb_queues %u",
 974                                                         dev, nb_qpairs);
 975                 return -EINVAL;
 976         }
 977
 978         CDEV_LOG_DEBUG("Setup %d queues pairs on device %u",
 979                         nb_qpairs, dev->data->dev_id);
 980
 981         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 982
 983         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 984         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 985
 986         if (nb_qpairs > (dev_info.max_nb_queue_pairs)) {
 987                 CDEV_LOG_ERR("Invalid num queue_pairs (%u) for dev %u",
 988                                 nb_qpairs, dev->data->dev_id);
 989             return -EINVAL;
 990         }
 991
 992         if (dev->data->queue_pairs == NULL) { /* first time configuration */
 993                 dev->data->queue_pairs = rte_zmalloc_socket(
 994                                 "cryptodev->queue_pairs",
 995                                 sizeof(dev->data->queue_pairs[0]) *
 996                                 dev_info.max_nb_queue_pairs,
 997                                 RTE_CACHE_LINE_SIZE, socket_id);
 998
 999                 if (dev->data->queue_pairs == NULL) {
1000                         dev->data->nb_queue_pairs = 0;
1001                         CDEV_LOG_ERR("failed to get memory for qp meta data, "
1002                                                         "nb_queues %u",
1003                                                         nb_qpairs);
1004                         return -(ENOMEM);
1005                 }
1006         } else { /* re-configure */
1007                 int ret;
1008                 uint16_t old_nb_queues = dev->data->nb_queue_pairs;
1009
1010                 qp = dev->data->queue_pairs;
1011
1012                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
1013                                 -ENOTSUP);
1014
1015                 for (i = nb_qpairs; i < old_nb_queues; i++) {
1016                         ret = (*dev->dev_ops->queue_pair_release)(dev, i);
1017                         if (ret < 0)
1018                                 return ret;
1019                         qp[i] = NULL;
1020                 }
1021
1022         }
1023         dev->data->nb_queue_pairs = nb_qpairs;
1024         return 0;
1025 }
1026
1027 int
1028 rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
1029 {
1030         struct rte_cryptodev *dev;
1031         int diag;
1032
1033         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1034                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1035                 return -EINVAL;
1036         }
1037
1038         dev = &rte_crypto_devices[dev_id];
1039
1040         if (dev->data->dev_started) {
1041                 CDEV_LOG_ERR(
1042                     "device %d must be stopped to allow configuration", dev_id);
1043                 return -EBUSY;
1044         }
1045
1046         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1047
1048         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1049         cryptodev_cb_cleanup(dev);
1050         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1051
1052         /* Setup new number of queue pairs and reconfigure device. */
1053         diag = rte_cryptodev_queue_pairs_config(dev, config->nb_queue_pairs,
1054                         config->socket_id);
1055         if (diag != 0) {
1056                 CDEV_LOG_ERR("dev%d rte_crypto_dev_queue_pairs_config = %d",
1057                                 dev_id, diag);
1058                 return diag;
1059         }
1060
1061         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1062         diag = cryptodev_cb_init(dev);
1063         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1064         if (diag) {
1065                 CDEV_LOG_ERR("Callback init failed for dev_id=%d", dev_id);
1066                 return diag;
1067         }
1068
1069         rte_cryptodev_trace_configure(dev_id, config);
1070         return (*dev->dev_ops->dev_configure)(dev, config);
1071 }
1072
1073 int
1074 rte_cryptodev_start(uint8_t dev_id)
1075 {
1076         struct rte_cryptodev *dev;
1077         int diag;
1078
1079         CDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id);
1080
1081         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1082                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1083                 return -EINVAL;
1084         }
1085
1086         dev = &rte_crypto_devices[dev_id];
1087
1088         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1089
1090         if (dev->data->dev_started != 0) {
1091                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already started",
1092                         dev_id);
1093                 return 0;
1094         }
1095
1096         diag = (*dev->dev_ops->dev_start)(dev);
1097         /* expose selection of PMD fast-path functions */
1098         cryptodev_fp_ops_set(rte_crypto_fp_ops + dev_id, dev);
1099
1100         rte_cryptodev_trace_start(dev_id, diag);
1101         if (diag == 0)
1102                 dev->data->dev_started = 1;
1103         else
1104                 return diag;
1105
1106         return 0;
1107 }
1108
1109 void
1110 rte_cryptodev_stop(uint8_t dev_id)
1111 {
1112         struct rte_cryptodev *dev;
1113
1114         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1115                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1116                 return;
1117         }
1118
1119         dev = &rte_crypto_devices[dev_id];
1120
1121         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1122
1123         if (dev->data->dev_started == 0) {
1124                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already stopped",
1125                         dev_id);
1126                 return;
1127         }
1128
1129         /* point fast-path functions to dummy ones */
1130         cryptodev_fp_ops_reset(rte_crypto_fp_ops + dev_id);
1131
1132         (*dev->dev_ops->dev_stop)(dev);
1133         rte_cryptodev_trace_stop(dev_id);
1134         dev->data->dev_started = 0;
1135 }
1136
1137 int
1138 rte_cryptodev_close(uint8_t dev_id)
1139 {
1140         struct rte_cryptodev *dev;
1141         int retval;
1142
1143         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1144                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1145                 return -1;
1146         }
1147
1148         dev = &rte_crypto_devices[dev_id];
1149
1150         /* Device must be stopped before it can be closed */
1151         if (dev->data->dev_started == 1) {
1152                 CDEV_LOG_ERR("Device %u must be stopped before closing",
1153                                 dev_id);
1154                 return -EBUSY;
1155         }
1156
1157         /* We can't close the device if there are outstanding sessions in use */
1158         if (dev->data->session_pool != NULL) {
1159                 if (!rte_mempool_full(dev->data->session_pool)) {
1160                         CDEV_LOG_ERR("dev_id=%u close failed, session mempool "
1161                                         "has sessions still in use, free "
1162                                         "all sessions before calling close",
1163                                         (unsigned)dev_id);
1164                         return -EBUSY;
1165                 }
1166         }
1167
1168         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
1169         retval = (*dev->dev_ops->dev_close)(dev);
1170         rte_cryptodev_trace_close(dev_id, retval);
1171
1172         if (retval < 0)
1173                 return retval;
1174
1175         return 0;
1176 }
1177
1178 int
1179 rte_cryptodev_get_qp_status(uint8_t dev_id, uint16_t queue_pair_id)
1180 {
1181         struct rte_cryptodev *dev;
1182
1183         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1184                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1185                 return -EINVAL;
1186         }
1187
1188         dev = &rte_crypto_devices[dev_id];
1189         if (queue_pair_id >= dev->data->nb_queue_pairs) {
1190                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
1191                 return -EINVAL;
1192         }
1193         void **qps = dev->data->queue_pairs;
1194
1195         if (qps[queue_pair_id]) {
1196                 CDEV_LOG_DEBUG("qp %d on dev %d is initialised",
1197                         queue_pair_id, dev_id);
1198                 return 1;
1199         }
1200
1201         CDEV_LOG_DEBUG("qp %d on dev %d is not initialised",
1202                 queue_pair_id, dev_id);
1203
1204         return 0;
1205 }
1206
1207 int
1208 rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
1209                 const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
1210
1211 {
1212         struct rte_cryptodev *dev;
1213
1214         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1215                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1216                 return -EINVAL;
1217         }
1218
1219         dev = &rte_crypto_devices[dev_id];
1220         if (queue_pair_id >= dev->data->nb_queue_pairs) {
1221                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
1222                 return -EINVAL;
1223         }
1224
1225         if (!qp_conf) {
1226                 CDEV_LOG_ERR("qp_conf cannot be NULL\n");
1227                 return -EINVAL;
1228         }
1229
1230         if ((qp_conf->mp_session && !qp_conf->mp_session_private) ||
1231                         (!qp_conf->mp_session && qp_conf->mp_session_private)) {
1232                 CDEV_LOG_ERR("Invalid mempools\n");
1233                 return -EINVAL;
1234         }
1235
1236         if (qp_conf->mp_session) {
1237                 struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1238                 uint32_t obj_size = qp_conf->mp_session->elt_size;
1239                 uint32_t obj_priv_size = qp_conf->mp_session_private->elt_size;
1240                 struct rte_cryptodev_sym_session s = {0};
1241
1242                 pool_priv = rte_mempool_get_priv(qp_conf->mp_session);
1243                 if (!pool_priv || qp_conf->mp_session->private_data_size <
1244                                 sizeof(*pool_priv)) {
1245                         CDEV_LOG_ERR("Invalid mempool\n");
1246                         return -EINVAL;
1247                 }
1248
1249                 s.nb_drivers = pool_priv->nb_drivers;
1250                 s.user_data_sz = pool_priv->user_data_sz;
1251
1252                 if ((rte_cryptodev_sym_get_existing_header_session_size(&s) >
1253                         obj_size) || (s.nb_drivers <= dev->driver_id) ||
1254                         rte_cryptodev_sym_get_private_session_size(dev_id) >
1255                                 obj_priv_size) {
1256                         CDEV_LOG_ERR("Invalid mempool\n");
1257                         return -EINVAL;
1258                 }
1259         }
1260
1261         if (dev->data->dev_started) {
1262                 CDEV_LOG_ERR(
1263                     "device %d must be stopped to allow configuration", dev_id);
1264                 return -EBUSY;
1265         }
1266
1267         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
1268
1269         rte_cryptodev_trace_queue_pair_setup(dev_id, queue_pair_id, qp_conf);
1270         return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id, qp_conf,
1271                         socket_id);
1272 }
1273
1274 struct rte_cryptodev_cb *
1275 rte_cryptodev_add_enq_callback(uint8_t dev_id,
1276                                uint16_t qp_id,
1277                                rte_cryptodev_callback_fn cb_fn,
1278                                void *cb_arg)
1279 {
1280         struct rte_cryptodev *dev;
1281         struct rte_cryptodev_cb_rcu *list;
1282         struct rte_cryptodev_cb *cb, *tail;
1283
1284         if (!cb_fn) {
1285                 CDEV_LOG_ERR("Callback is NULL on dev_id=%d", dev_id);
1286                 rte_errno = EINVAL;
1287                 return NULL;
1288         }
1289
1290         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1291                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1292                 rte_errno = ENODEV;
1293                 return NULL;
1294         }
1295
1296         dev = &rte_crypto_devices[dev_id];
1297         if (qp_id >= dev->data->nb_queue_pairs) {
1298                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1299                 rte_errno = ENODEV;
1300                 return NULL;
1301         }
1302
1303         cb = rte_zmalloc(NULL, sizeof(*cb), 0);
1304         if (cb == NULL) {
1305                 CDEV_LOG_ERR("Failed to allocate memory for callback on "
1306                              "dev=%d, queue_pair_id=%d", dev_id, qp_id);
1307                 rte_errno = ENOMEM;
1308                 return NULL;
1309         }
1310
1311         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1312
1313         cb->fn = cb_fn;
1314         cb->arg = cb_arg;
1315
1316         /* Add the callbacks in fifo order. */
1317         list = &dev->enq_cbs[qp_id];
1318         tail = list->next;
1319
1320         if (tail) {
1321                 while (tail->next)
1322                         tail = tail->next;
1323                 /* Stores to cb->fn and cb->param should complete before
1324                  * cb is visible to data plane.
1325                  */
1326                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
1327         } else {
1328                 /* Stores to cb->fn and cb->param should complete before
1329                  * cb is visible to data plane.
1330                  */
1331                 __atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
1332         }
1333
1334         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1335
1336         return cb;
1337 }
1338
1339 int
1340 rte_cryptodev_remove_enq_callback(uint8_t dev_id,
1341                                   uint16_t qp_id,
1342                                   struct rte_cryptodev_cb *cb)
1343 {
1344         struct rte_cryptodev *dev;
1345         struct rte_cryptodev_cb **prev_cb, *curr_cb;
1346         struct rte_cryptodev_cb_rcu *list;
1347         int ret;
1348
1349         ret = -EINVAL;
1350
1351         if (!cb) {
1352                 CDEV_LOG_ERR("Callback is NULL");
1353                 return -EINVAL;
1354         }
1355
1356         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1357                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1358                 return -ENODEV;
1359         }
1360
1361         dev = &rte_crypto_devices[dev_id];
1362         if (qp_id >= dev->data->nb_queue_pairs) {
1363                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1364                 return -ENODEV;
1365         }
1366
1367         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1368         if (dev->enq_cbs == NULL) {
1369                 CDEV_LOG_ERR("Callback not initialized");
1370                 goto cb_err;
1371         }
1372
1373         list = &dev->enq_cbs[qp_id];
1374         if (list == NULL) {
1375                 CDEV_LOG_ERR("Callback list is NULL");
1376                 goto cb_err;
1377         }
1378
1379         if (list->qsbr == NULL) {
1380                 CDEV_LOG_ERR("Rcu qsbr is NULL");
1381                 goto cb_err;
1382         }
1383
1384         prev_cb = &list->next;
1385         for (; *prev_cb != NULL; prev_cb = &curr_cb->next) {
1386                 curr_cb = *prev_cb;
1387                 if (curr_cb == cb) {
1388                         /* Remove the user cb from the callback list. */
1389                         __atomic_store_n(prev_cb, curr_cb->next,
1390                                 __ATOMIC_RELAXED);
1391                         ret = 0;
1392                         break;
1393                 }
1394         }
1395
1396         if (!ret) {
1397                 /* Call sync with invalid thread id as this is part of
1398                  * control plane API
1399                  */
1400                 rte_rcu_qsbr_synchronize(list->qsbr, RTE_QSBR_THRID_INVALID);
1401                 rte_free(cb);
1402         }
1403
1404 cb_err:
1405         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1406         return ret;
1407 }
1408
1409 struct rte_cryptodev_cb *
1410 rte_cryptodev_add_deq_callback(uint8_t dev_id,
1411                                uint16_t qp_id,
1412                                rte_cryptodev_callback_fn cb_fn,
1413                                void *cb_arg)
1414 {
1415         struct rte_cryptodev *dev;
1416         struct rte_cryptodev_cb_rcu *list;
1417         struct rte_cryptodev_cb *cb, *tail;
1418
1419         if (!cb_fn) {
1420                 CDEV_LOG_ERR("Callback is NULL on dev_id=%d", dev_id);
1421                 rte_errno = EINVAL;
1422                 return NULL;
1423         }
1424
1425         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1426                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1427                 rte_errno = ENODEV;
1428                 return NULL;
1429         }
1430
1431         dev = &rte_crypto_devices[dev_id];
1432         if (qp_id >= dev->data->nb_queue_pairs) {
1433                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1434                 rte_errno = ENODEV;
1435                 return NULL;
1436         }
1437
1438         cb = rte_zmalloc(NULL, sizeof(*cb), 0);
1439         if (cb == NULL) {
1440                 CDEV_LOG_ERR("Failed to allocate memory for callback on "
1441                              "dev=%d, queue_pair_id=%d", dev_id, qp_id);
1442                 rte_errno = ENOMEM;
1443                 return NULL;
1444         }
1445
1446         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1447
1448         cb->fn = cb_fn;
1449         cb->arg = cb_arg;
1450
1451         /* Add the callbacks in fifo order. */
1452         list = &dev->deq_cbs[qp_id];
1453         tail = list->next;
1454
1455         if (tail) {
1456                 while (tail->next)
1457                         tail = tail->next;
1458                 /* Stores to cb->fn and cb->param should complete before
1459                  * cb is visible to data plane.
1460                  */
1461                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
1462         } else {
1463                 /* Stores to cb->fn and cb->param should complete before
1464                  * cb is visible to data plane.
1465                  */
1466                 __atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
1467         }
1468
1469         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1470
1471         return cb;
1472 }
1473
1474 int
1475 rte_cryptodev_remove_deq_callback(uint8_t dev_id,
1476                                   uint16_t qp_id,
1477                                   struct rte_cryptodev_cb *cb)
1478 {
1479         struct rte_cryptodev *dev;
1480         struct rte_cryptodev_cb **prev_cb, *curr_cb;
1481         struct rte_cryptodev_cb_rcu *list;
1482         int ret;
1483
1484         ret = -EINVAL;
1485
1486         if (!cb) {
1487                 CDEV_LOG_ERR("Callback is NULL");
1488                 return -EINVAL;
1489         }
1490
1491         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1492                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1493                 return -ENODEV;
1494         }
1495
1496         dev = &rte_crypto_devices[dev_id];
1497         if (qp_id >= dev->data->nb_queue_pairs) {
1498                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1499                 return -ENODEV;
1500         }
1501
1502         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1503         if (dev->enq_cbs == NULL) {
1504                 CDEV_LOG_ERR("Callback not initialized");
1505                 goto cb_err;
1506         }
1507
1508         list = &dev->deq_cbs[qp_id];
1509         if (list == NULL) {
1510                 CDEV_LOG_ERR("Callback list is NULL");
1511                 goto cb_err;
1512         }
1513
1514         if (list->qsbr == NULL) {
1515                 CDEV_LOG_ERR("Rcu qsbr is NULL");
1516                 goto cb_err;
1517         }
1518
1519         prev_cb = &list->next;
1520         for (; *prev_cb != NULL; prev_cb = &curr_cb->next) {
1521                 curr_cb = *prev_cb;
1522                 if (curr_cb == cb) {
1523                         /* Remove the user cb from the callback list. */
1524                         __atomic_store_n(prev_cb, curr_cb->next,
1525                                 __ATOMIC_RELAXED);
1526                         ret = 0;
1527                         break;
1528                 }
1529         }
1530
1531         if (!ret) {
1532                 /* Call sync with invalid thread id as this is part of
1533                  * control plane API
1534                  */
1535                 rte_rcu_qsbr_synchronize(list->qsbr, RTE_QSBR_THRID_INVALID);
1536                 rte_free(cb);
1537         }
1538
1539 cb_err:
1540         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1541         return ret;
1542 }
1543
1544 int
1545 rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats)
1546 {
1547         struct rte_cryptodev *dev;
1548
1549         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1550                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1551                 return -ENODEV;
1552         }
1553
1554         if (stats == NULL) {
1555                 CDEV_LOG_ERR("Invalid stats ptr");
1556                 return -EINVAL;
1557         }
1558
1559         dev = &rte_crypto_devices[dev_id];
1560         memset(stats, 0, sizeof(*stats));
1561
1562         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1563         (*dev->dev_ops->stats_get)(dev, stats);
1564         return 0;
1565 }
1566
1567 void
1568 rte_cryptodev_stats_reset(uint8_t dev_id)
1569 {
1570         struct rte_cryptodev *dev;
1571
1572         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1573                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1574                 return;
1575         }
1576
1577         dev = &rte_crypto_devices[dev_id];
1578
1579         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
1580         (*dev->dev_ops->stats_reset)(dev);
1581 }
1582
1583 void
1584 rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
1585 {
1586         struct rte_cryptodev *dev;
1587
1588         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1589                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1590                 return;
1591         }
1592
1593         dev = &rte_crypto_devices[dev_id];
1594
1595         memset(dev_info, 0, sizeof(struct rte_cryptodev_info));
1596
1597         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
1598         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
1599
1600         dev_info->driver_name = dev->device->driver->name;
1601         dev_info->device = dev->device;
1602 }
1603
1604 int
1605 rte_cryptodev_callback_register(uint8_t dev_id,
1606                         enum rte_cryptodev_event_type event,
1607                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
1608 {
1609         struct rte_cryptodev *dev;
1610         struct rte_cryptodev_callback *user_cb;
1611
1612         if (!cb_fn)
1613                 return -EINVAL;
1614
1615         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1616                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1617                 return -EINVAL;
1618         }
1619
1620         dev = &rte_crypto_devices[dev_id];
1621         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1622
1623         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
1624                 if (user_cb->cb_fn == cb_fn &&
1625                         user_cb->cb_arg == cb_arg &&
1626                         user_cb->event == event) {
1627                         break;
1628                 }
1629         }
1630
1631         /* create a new callback. */
1632         if (user_cb == NULL) {
1633                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
1634                                 sizeof(struct rte_cryptodev_callback), 0);
1635                 if (user_cb != NULL) {
1636                         user_cb->cb_fn = cb_fn;
1637                         user_cb->cb_arg = cb_arg;
1638                         user_cb->event = event;
1639                         TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
1640                 }
1641         }
1642
1643         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1644         return (user_cb == NULL) ? -ENOMEM : 0;
1645 }
1646
1647 int
1648 rte_cryptodev_callback_unregister(uint8_t dev_id,
1649                         enum rte_cryptodev_event_type event,
1650                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
1651 {
1652         int ret;
1653         struct rte_cryptodev *dev;
1654         struct rte_cryptodev_callback *cb, *next;
1655
1656         if (!cb_fn)
1657                 return -EINVAL;
1658
1659         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1660                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1661                 return -EINVAL;
1662         }
1663
1664         dev = &rte_crypto_devices[dev_id];
1665         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1666
1667         ret = 0;
1668         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
1669
1670                 next = TAILQ_NEXT(cb, next);
1671
1672                 if (cb->cb_fn != cb_fn || cb->event != event ||
1673                                 (cb->cb_arg != (void *)-1 &&
1674                                 cb->cb_arg != cb_arg))
1675                         continue;
1676
1677                 /*
1678                  * if this callback is not executing right now,
1679                  * then remove it.
1680                  */
1681                 if (cb->active == 0) {
1682                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
1683                         rte_free(cb);
1684                 } else {
1685                         ret = -EAGAIN;
1686                 }
1687         }
1688
1689         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1690         return ret;
1691 }
1692
1693 void
1694 rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev,
1695         enum rte_cryptodev_event_type event)
1696 {
1697         struct rte_cryptodev_callback *cb_lst;
1698         struct rte_cryptodev_callback dev_cb;
1699
1700         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1701         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
1702                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1703                         continue;
1704                 dev_cb = *cb_lst;
1705                 cb_lst->active = 1;
1706                 rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1707                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1708                                                 dev_cb.cb_arg);
1709                 rte_spinlock_lock(&rte_cryptodev_cb_lock);
1710                 cb_lst->active = 0;
1711         }
1712         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1713 }
1714
1715 int
1716 rte_cryptodev_sym_session_init(uint8_t dev_id,
1717                 struct rte_cryptodev_sym_session *sess,
1718                 struct rte_crypto_sym_xform *xforms,
1719                 struct rte_mempool *mp)
1720 {
1721         struct rte_cryptodev *dev;
1722         uint32_t sess_priv_sz = rte_cryptodev_sym_get_private_session_size(
1723                         dev_id);
1724         uint8_t index;
1725         int ret;
1726
1727         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1728                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1729                 return -EINVAL;
1730         }
1731
1732         dev = rte_cryptodev_pmd_get_dev(dev_id);
1733
1734         if (sess == NULL || xforms == NULL || dev == NULL || mp == NULL)
1735                 return -EINVAL;
1736
1737         if (mp->elt_size < sess_priv_sz)
1738                 return -EINVAL;
1739
1740         index = dev->driver_id;
1741         if (index >= sess->nb_drivers)
1742                 return -EINVAL;
1743
1744         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_configure, -ENOTSUP);
1745
1746         if (sess->sess_data[index].refcnt == 0) {
1747                 ret = dev->dev_ops->sym_session_configure(dev, xforms,
1748                                                         sess, mp);
1749                 if (ret < 0) {
1750                         CDEV_LOG_ERR(
1751                                 "dev_id %d failed to configure session details",
1752                                 dev_id);
1753                         return ret;
1754                 }
1755         }
1756
1757         rte_cryptodev_trace_sym_session_init(dev_id, sess, xforms, mp);
1758         sess->sess_data[index].refcnt++;
1759         return 0;
1760 }
1761
1762 struct rte_mempool *
1763 rte_cryptodev_sym_session_pool_create(const char *name, uint32_t nb_elts,
1764         uint32_t elt_size, uint32_t cache_size, uint16_t user_data_size,
1765         int socket_id)
1766 {
1767         struct rte_mempool *mp;
1768         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1769         uint32_t obj_sz;
1770
1771         obj_sz = rte_cryptodev_sym_get_header_session_size() + user_data_size;
1772         if (obj_sz > elt_size)
1773                 CDEV_LOG_INFO("elt_size %u is expanded to %u\n", elt_size,
1774                                 obj_sz);
1775         else
1776                 obj_sz = elt_size;
1777
1778         mp = rte_mempool_create(name, nb_elts, obj_sz, cache_size,
1779                         (uint32_t)(sizeof(*pool_priv)),
1780                         NULL, NULL, NULL, NULL,
1781                         socket_id, 0);
1782         if (mp == NULL) {
1783                 CDEV_LOG_ERR("%s(name=%s) failed, rte_errno=%d\n",
1784                         __func__, name, rte_errno);
1785                 return NULL;
1786         }
1787
1788         pool_priv = rte_mempool_get_priv(mp);
1789         if (!pool_priv) {
1790                 CDEV_LOG_ERR("%s(name=%s) failed to get private data\n",
1791                         __func__, name);
1792                 rte_mempool_free(mp);
1793                 return NULL;
1794         }
1795
1796         pool_priv->nb_drivers = nb_drivers;
1797         pool_priv->user_data_sz = user_data_size;
1798
1799         rte_cryptodev_trace_sym_session_pool_create(name, nb_elts,
1800                 elt_size, cache_size, user_data_size, mp);
1801         return mp;
1802 }
1803
1804 struct rte_mempool *
1805 rte_cryptodev_asym_session_pool_create(const char *name, uint32_t nb_elts,
1806         uint32_t cache_size, int socket_id)
1807 {
1808         struct rte_mempool *mp;
1809         struct rte_cryptodev_asym_session_pool_private_data *pool_priv;
1810         uint32_t obj_sz, obj_sz_aligned;
1811         uint8_t dev_id, priv_sz, max_priv_sz = 0;
1812
1813         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++)
1814                 if (rte_cryptodev_is_valid_dev(dev_id)) {
1815                         priv_sz = rte_cryptodev_asym_get_private_session_size(dev_id);
1816                         if (priv_sz > max_priv_sz)
1817                                 max_priv_sz = priv_sz;
1818                 }
1819         if (max_priv_sz == 0) {
1820                 CDEV_LOG_INFO("Could not set max private session size\n");
1821                 return NULL;
1822         }
1823
1824         obj_sz = rte_cryptodev_asym_get_header_session_size() + max_priv_sz;
1825         obj_sz_aligned =  RTE_ALIGN_CEIL(obj_sz, RTE_CACHE_LINE_SIZE);
1826
1827         mp = rte_mempool_create(name, nb_elts, obj_sz_aligned, cache_size,
1828                         (uint32_t)(sizeof(*pool_priv)),
1829                         NULL, NULL, NULL, NULL,
1830                         socket_id, 0);
1831         if (mp == NULL) {
1832                 CDEV_LOG_ERR("%s(name=%s) failed, rte_errno=%d\n",
1833                         __func__, name, rte_errno);
1834                 return NULL;
1835         }
1836
1837         pool_priv = rte_mempool_get_priv(mp);
1838         if (!pool_priv) {
1839                 CDEV_LOG_ERR("%s(name=%s) failed to get private data\n",
1840                         __func__, name);
1841                 rte_mempool_free(mp);
1842                 return NULL;
1843         }
1844         pool_priv->max_priv_session_sz = max_priv_sz;
1845
1846         rte_cryptodev_trace_asym_session_pool_create(name, nb_elts,
1847                 cache_size, mp);
1848         return mp;
1849 }
1850
1851 static unsigned int
1852 rte_cryptodev_sym_session_data_size(struct rte_cryptodev_sym_session *sess)
1853 {
1854         return (sizeof(sess->sess_data[0]) * sess->nb_drivers) +
1855                         sess->user_data_sz;
1856 }
1857
1858 static uint8_t
1859 rte_cryptodev_sym_is_valid_session_pool(struct rte_mempool *mp)
1860 {
1861         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1862
1863         if (!mp)
1864                 return 0;
1865
1866         pool_priv = rte_mempool_get_priv(mp);
1867
1868         if (!pool_priv || mp->private_data_size < sizeof(*pool_priv) ||
1869                         pool_priv->nb_drivers != nb_drivers ||
1870                         mp->elt_size <
1871                                 rte_cryptodev_sym_get_header_session_size()
1872                                 + pool_priv->user_data_sz)
1873                 return 0;
1874
1875         return 1;
1876 }
1877
1878 struct rte_cryptodev_sym_session *
1879 rte_cryptodev_sym_session_create(struct rte_mempool *mp)
1880 {
1881         struct rte_cryptodev_sym_session *sess;
1882         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1883
1884         if (!rte_cryptodev_sym_is_valid_session_pool(mp)) {
1885                 CDEV_LOG_ERR("Invalid mempool\n");
1886                 return NULL;
1887         }
1888
1889         pool_priv = rte_mempool_get_priv(mp);
1890
1891         /* Allocate a session structure from the session pool */
1892         if (rte_mempool_get(mp, (void **)&sess)) {
1893                 CDEV_LOG_ERR("couldn't get object from session mempool");
1894                 return NULL;
1895         }
1896
1897         sess->nb_drivers = pool_priv->nb_drivers;
1898         sess->user_data_sz = pool_priv->user_data_sz;
1899         sess->opaque_data = 0;
1900
1901         /* Clear device session pointer.
1902          * Include the flag indicating presence of user data
1903          */
1904         memset(sess->sess_data, 0,
1905                         rte_cryptodev_sym_session_data_size(sess));
1906
1907         rte_cryptodev_trace_sym_session_create(mp, sess);
1908         return sess;
1909 }
1910
1911 void *
1912 rte_cryptodev_asym_session_create(uint8_t dev_id,
1913                 struct rte_crypto_asym_xform *xforms, struct rte_mempool *mp)
1914 {
1915         struct rte_cryptodev_asym_session *sess;
1916         uint32_t session_priv_data_sz;
1917         struct rte_cryptodev_asym_session_pool_private_data *pool_priv;
1918         unsigned int session_header_size =
1919                         rte_cryptodev_asym_get_header_session_size();
1920         struct rte_cryptodev *dev;
1921         int ret;
1922
1923         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1924                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1925                 return NULL;
1926         }
1927
1928         dev = rte_cryptodev_pmd_get_dev(dev_id);
1929
1930         if (dev == NULL)
1931                 return NULL;
1932
1933         if (!mp) {
1934                 CDEV_LOG_ERR("invalid mempool\n");
1935                 return NULL;
1936         }
1937
1938         session_priv_data_sz = rte_cryptodev_asym_get_private_session_size(
1939                         dev_id);
1940         pool_priv = rte_mempool_get_priv(mp);
1941
1942         if (pool_priv->max_priv_session_sz < session_priv_data_sz) {
1943                 CDEV_LOG_DEBUG(
1944                         "The private session data size used when creating the mempool is smaller than this device's private session data.");
1945                 return NULL;
1946         }
1947
1948         /* Verify if provided mempool can hold elements big enough. */
1949         if (mp->elt_size < session_header_size + session_priv_data_sz) {
1950                 CDEV_LOG_ERR(
1951                         "mempool elements too small to hold session objects");
1952                 return NULL;
1953         }
1954
1955         /* Allocate a session structure from the session pool */
1956         if (rte_mempool_get(mp, (void **)&sess)) {
1957                 CDEV_LOG_ERR("couldn't get object from session mempool");
1958                 return NULL;
1959         }
1960
1961         sess->driver_id = dev->driver_id;
1962         sess->max_priv_data_sz = pool_priv->max_priv_session_sz;
1963
1964         /* Clear device session pointer.*/
1965         memset(sess->sess_private_data, 0, session_priv_data_sz);
1966
1967         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_configure, NULL);
1968
1969         if (sess->sess_private_data[0] == 0) {
1970                 ret = dev->dev_ops->asym_session_configure(dev, xforms, sess);
1971                 if (ret < 0) {
1972                         CDEV_LOG_ERR(
1973                                 "dev_id %d failed to configure session details",
1974                                 dev_id);
1975                         return NULL;
1976                 }
1977         }
1978
1979         rte_cryptodev_trace_asym_session_create(dev_id, xforms, mp);
1980         return sess;
1981 }
1982
1983 int
1984 rte_cryptodev_sym_session_clear(uint8_t dev_id,
1985                 struct rte_cryptodev_sym_session *sess)
1986 {
1987         struct rte_cryptodev *dev;
1988         uint8_t driver_id;
1989
1990         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1991                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1992                 return -EINVAL;
1993         }
1994
1995         dev = rte_cryptodev_pmd_get_dev(dev_id);
1996
1997         if (dev == NULL || sess == NULL)
1998                 return -EINVAL;
1999
2000         driver_id = dev->driver_id;
2001         if (sess->sess_data[driver_id].refcnt == 0)
2002                 return 0;
2003         if (--sess->sess_data[driver_id].refcnt != 0)
2004                 return -EBUSY;
2005
2006         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_clear, -ENOTSUP);
2007
2008         dev->dev_ops->sym_session_clear(dev, sess);
2009
2010         rte_cryptodev_trace_sym_session_clear(dev_id, sess);
2011         return 0;
2012 }
2013
2014 int
2015 rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess)
2016 {
2017         uint8_t i;
2018         struct rte_mempool *sess_mp;
2019
2020         if (sess == NULL)
2021                 return -EINVAL;
2022
2023         /* Check that all device private data has been freed */
2024         for (i = 0; i < sess->nb_drivers; i++) {
2025                 if (sess->sess_data[i].refcnt != 0)
2026                         return -EBUSY;
2027         }
2028
2029         /* Return session to mempool */
2030         sess_mp = rte_mempool_from_obj(sess);
2031         rte_mempool_put(sess_mp, sess);
2032
2033         rte_cryptodev_trace_sym_session_free(sess);
2034         return 0;
2035 }
2036
2037 int
2038 rte_cryptodev_asym_session_free(uint8_t dev_id, void *sess)
2039 {
2040         struct rte_mempool *sess_mp;
2041         struct rte_cryptodev *dev;
2042
2043         if (!rte_cryptodev_is_valid_dev(dev_id)) {
2044                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
2045                 return -EINVAL;
2046         }
2047
2048         dev = rte_cryptodev_pmd_get_dev(dev_id);
2049
2050         if (dev == NULL || sess == NULL)
2051                 return -EINVAL;
2052
2053         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_clear, -ENOTSUP);
2054
2055         dev->dev_ops->asym_session_clear(dev, sess);
2056
2057         /* Return session to mempool */
2058         sess_mp = rte_mempool_from_obj(sess);
2059         rte_mempool_put(sess_mp, sess);
2060
2061         rte_cryptodev_trace_asym_session_free(dev_id, sess);
2062         return 0;
2063 }
2064
2065 unsigned int
2066 rte_cryptodev_sym_get_header_session_size(void)
2067 {
2068         /*
2069          * Header contains pointers to the private data of all registered
2070          * drivers and all necessary information to ensure safely clear
2071          * or free al session.
2072          */
2073         struct rte_cryptodev_sym_session s = {0};
2074
2075         s.nb_drivers = nb_drivers;
2076
2077         return (unsigned int)(sizeof(s) +
2078                         rte_cryptodev_sym_session_data_size(&s));
2079 }
2080
2081 unsigned int
2082 rte_cryptodev_sym_get_existing_header_session_size(
2083                 struct rte_cryptodev_sym_session *sess)
2084 {
2085         if (!sess)
2086                 return 0;
2087         else
2088                 return (unsigned int)(sizeof(*sess) +
2089                                 rte_cryptodev_sym_session_data_size(sess));
2090 }
2091
2092 unsigned int
2093 rte_cryptodev_asym_get_header_session_size(void)
2094 {
2095         return sizeof(struct rte_cryptodev_asym_session);
2096 }
2097
2098 unsigned int
2099 rte_cryptodev_sym_get_private_session_size(uint8_t dev_id)
2100 {
2101         struct rte_cryptodev *dev;
2102         unsigned int priv_sess_size;
2103
2104         if (!rte_cryptodev_is_valid_dev(dev_id))
2105                 return 0;
2106
2107         dev = rte_cryptodev_pmd_get_dev(dev_id);
2108
2109         if (*dev->dev_ops->sym_session_get_size == NULL)
2110                 return 0;
2111
2112         priv_sess_size = (*dev->dev_ops->sym_session_get_size)(dev);
2113
2114         return priv_sess_size;
2115 }
2116
2117 unsigned int
2118 rte_cryptodev_asym_get_private_session_size(uint8_t dev_id)
2119 {
2120         struct rte_cryptodev *dev;
2121         unsigned int priv_sess_size;
2122
2123         if (!rte_cryptodev_is_valid_dev(dev_id))
2124                 return 0;
2125
2126         dev = rte_cryptodev_pmd_get_dev(dev_id);
2127
2128         if (*dev->dev_ops->asym_session_get_size == NULL)
2129                 return 0;
2130
2131         priv_sess_size = (*dev->dev_ops->asym_session_get_size)(dev);
2132
2133         return priv_sess_size;
2134 }
2135
2136 int
2137 rte_cryptodev_sym_session_set_user_data(
2138                                         struct rte_cryptodev_sym_session *sess,
2139                                         void *data,
2140                                         uint16_t size)
2141 {
2142         if (sess == NULL)
2143                 return -EINVAL;
2144
2145         if (sess->user_data_sz < size)
2146                 return -ENOMEM;
2147
2148         rte_memcpy(sess->sess_data + sess->nb_drivers, data, size);
2149         return 0;
2150 }
2151
2152 void *
2153 rte_cryptodev_sym_session_get_user_data(
2154                                         struct rte_cryptodev_sym_session *sess)
2155 {
2156         if (sess == NULL || sess->user_data_sz == 0)
2157                 return NULL;
2158
2159         return (void *)(sess->sess_data + sess->nb_drivers);
2160 }
2161
2162 static inline void
2163 sym_crypto_fill_status(struct rte_crypto_sym_vec *vec, int32_t errnum)
2164 {
2165         uint32_t i;
2166         for (i = 0; i < vec->num; i++)
2167                 vec->status[i] = errnum;
2168 }
2169
2170 uint32_t
2171 rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
2172         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs,
2173         struct rte_crypto_sym_vec *vec)
2174 {
2175         struct rte_cryptodev *dev;
2176
2177         if (!rte_cryptodev_is_valid_dev(dev_id)) {
2178                 sym_crypto_fill_status(vec, EINVAL);
2179                 return 0;
2180         }
2181
2182         dev = rte_cryptodev_pmd_get_dev(dev_id);
2183
2184         if (*dev->dev_ops->sym_cpu_process == NULL ||
2185                 !(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO)) {
2186                 sym_crypto_fill_status(vec, ENOTSUP);
2187                 return 0;
2188         }
2189
2190         return dev->dev_ops->sym_cpu_process(dev, sess, ofs, vec);
2191 }
2192
2193 int
2194 rte_cryptodev_get_raw_dp_ctx_size(uint8_t dev_id)
2195 {
2196         struct rte_cryptodev *dev;
2197         int32_t size = sizeof(struct rte_crypto_raw_dp_ctx);
2198         int32_t priv_size;
2199
2200         if (!rte_cryptodev_is_valid_dev(dev_id))
2201                 return -EINVAL;
2202
2203         dev = rte_cryptodev_pmd_get_dev(dev_id);
2204
2205         if (*dev->dev_ops->sym_get_raw_dp_ctx_size == NULL ||
2206                 !(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)) {
2207                 return -ENOTSUP;
2208         }
2209
2210         priv_size = (*dev->dev_ops->sym_get_raw_dp_ctx_size)(dev);
2211         if (priv_size < 0)
2212                 return -ENOTSUP;
2213
2214         return RTE_ALIGN_CEIL((size + priv_size), 8);
2215 }
2216
2217 int
2218 rte_cryptodev_configure_raw_dp_ctx(uint8_t dev_id, uint16_t qp_id,
2219         struct rte_crypto_raw_dp_ctx *ctx,
2220         enum rte_crypto_op_sess_type sess_type,
2221         union rte_cryptodev_session_ctx session_ctx,
2222         uint8_t is_update)
2223 {
2224         struct rte_cryptodev *dev;
2225
2226         if (!rte_cryptodev_get_qp_status(dev_id, qp_id))
2227                 return -EINVAL;
2228
2229         dev = rte_cryptodev_pmd_get_dev(dev_id);
2230         if (!(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)
2231                         || dev->dev_ops->sym_configure_raw_dp_ctx == NULL)
2232                 return -ENOTSUP;
2233
2234         return (*dev->dev_ops->sym_configure_raw_dp_ctx)(dev, qp_id, ctx,
2235                         sess_type, session_ctx, is_update);
2236 }
2237
2238 uint32_t
2239 rte_cryptodev_raw_enqueue_burst(struct rte_crypto_raw_dp_ctx *ctx,
2240         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
2241         void **user_data, int *enqueue_status)
2242 {
2243         return (*ctx->enqueue_burst)(ctx->qp_data, ctx->drv_ctx_data, vec,
2244                         ofs, user_data, enqueue_status);
2245 }
2246
2247 int
2248 rte_cryptodev_raw_enqueue_done(struct rte_crypto_raw_dp_ctx *ctx,
2249                 uint32_t n)
2250 {
2251         return (*ctx->enqueue_done)(ctx->qp_data, ctx->drv_ctx_data, n);
2252 }
2253
2254 uint32_t
2255 rte_cryptodev_raw_dequeue_burst(struct rte_crypto_raw_dp_ctx *ctx,
2256         rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
2257         uint32_t max_nb_to_dequeue,
2258         rte_cryptodev_raw_post_dequeue_t post_dequeue,
2259         void **out_user_data, uint8_t is_user_data_array,
2260         uint32_t *n_success_jobs, int *status)
2261 {
2262         return (*ctx->dequeue_burst)(ctx->qp_data, ctx->drv_ctx_data,
2263                 get_dequeue_count, max_nb_to_dequeue, post_dequeue,
2264                 out_user_data, is_user_data_array, n_success_jobs, status);
2265 }
2266
2267 int
2268 rte_cryptodev_raw_dequeue_done(struct rte_crypto_raw_dp_ctx *ctx,
2269                 uint32_t n)
2270 {
2271         return (*ctx->dequeue_done)(ctx->qp_data, ctx->drv_ctx_data, n);
2272 }
2273
2274 /** Initialise rte_crypto_op mempool element */
2275 static void
2276 rte_crypto_op_init(struct rte_mempool *mempool,
2277                 void *opaque_arg,
2278                 void *_op_data,
2279                 __rte_unused unsigned i)
2280 {
2281         struct rte_crypto_op *op = _op_data;
2282         enum rte_crypto_op_type type = *(enum rte_crypto_op_type *)opaque_arg;
2283
2284         memset(_op_data, 0, mempool->elt_size);
2285
2286         __rte_crypto_op_reset(op, type);
2287
2288         op->phys_addr = rte_mem_virt2iova(_op_data);
2289         op->mempool = mempool;
2290 }
2291
2292
2293 struct rte_mempool *
2294 rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type,
2295                 unsigned nb_elts, unsigned cache_size, uint16_t priv_size,
2296                 int socket_id)
2297 {
2298         struct rte_crypto_op_pool_private *priv;
2299
2300         unsigned elt_size = sizeof(struct rte_crypto_op) +
2301                         priv_size;
2302
2303         if (type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
2304                 elt_size += sizeof(struct rte_crypto_sym_op);
2305         } else if (type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
2306                 elt_size += sizeof(struct rte_crypto_asym_op);
2307         } else if (type == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
2308                 elt_size += RTE_MAX(sizeof(struct rte_crypto_sym_op),
2309                                     sizeof(struct rte_crypto_asym_op));
2310         } else {
2311                 CDEV_LOG_ERR("Invalid op_type\n");
2312                 return NULL;
2313         }
2314
2315         /* lookup mempool in case already allocated */
2316         struct rte_mempool *mp = rte_mempool_lookup(name);
2317
2318         if (mp != NULL) {
2319                 priv = (struct rte_crypto_op_pool_private *)
2320                                 rte_mempool_get_priv(mp);
2321
2322                 if (mp->elt_size != elt_size ||
2323                                 mp->cache_size < cache_size ||
2324                                 mp->size < nb_elts ||
2325                                 priv->priv_size <  priv_size) {
2326                         mp = NULL;
2327                         CDEV_LOG_ERR("Mempool %s already exists but with "
2328                                         "incompatible parameters", name);
2329                         return NULL;
2330                 }
2331                 return mp;
2332         }
2333
2334         mp = rte_mempool_create(
2335                         name,
2336                         nb_elts,
2337                         elt_size,
2338                         cache_size,
2339                         sizeof(struct rte_crypto_op_pool_private),
2340                         NULL,
2341                         NULL,
2342                         rte_crypto_op_init,
2343                         &type,
2344                         socket_id,
2345                         0);
2346
2347         if (mp == NULL) {
2348                 CDEV_LOG_ERR("Failed to create mempool %s", name);
2349                 return NULL;
2350         }
2351
2352         priv = (struct rte_crypto_op_pool_private *)
2353                         rte_mempool_get_priv(mp);
2354
2355         priv->priv_size = priv_size;
2356         priv->type = type;
2357
2358         return mp;
2359 }
2360
2361 int
2362 rte_cryptodev_pmd_create_dev_name(char *name, const char *dev_name_prefix)
2363 {
2364         struct rte_cryptodev *dev = NULL;
2365         uint32_t i = 0;
2366
2367         if (name == NULL)
2368                 return -EINVAL;
2369
2370         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
2371                 int ret = snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
2372                                 "%s_%u", dev_name_prefix, i);
2373
2374                 if (ret < 0)
2375                         return ret;
2376
2377                 dev = rte_cryptodev_pmd_get_named_dev(name);
2378                 if (!dev)
2379                         return 0;
2380         }
2381
2382         return -1;
2383 }
2384
2385 TAILQ_HEAD(cryptodev_driver_list, cryptodev_driver);
2386
2387 static struct cryptodev_driver_list cryptodev_driver_list =
2388         TAILQ_HEAD_INITIALIZER(cryptodev_driver_list);
2389
2390 int
2391 rte_cryptodev_driver_id_get(const char *name)
2392 {
2393         struct cryptodev_driver *driver;
2394         const char *driver_name;
2395
2396         if (name == NULL) {
2397                 RTE_LOG(DEBUG, CRYPTODEV, "name pointer NULL");
2398                 return -1;
2399         }
2400
2401         TAILQ_FOREACH(driver, &cryptodev_driver_list, next) {
2402                 driver_name = driver->driver->name;
2403                 if (strncmp(driver_name, name, strlen(driver_name) + 1) == 0)
2404                         return driver->id;
2405         }
2406         return -1;
2407 }
2408
2409 const char *
2410 rte_cryptodev_name_get(uint8_t dev_id)
2411 {
2412         struct rte_cryptodev *dev;
2413
2414         if (!rte_cryptodev_is_valid_device_data(dev_id)) {
2415                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
2416                 return NULL;
2417         }
2418
2419         dev = rte_cryptodev_pmd_get_dev(dev_id);
2420         if (dev == NULL)
2421                 return NULL;
2422
2423         return dev->data->name;
2424 }
2425
2426 const char *
2427 rte_cryptodev_driver_name_get(uint8_t driver_id)
2428 {
2429         struct cryptodev_driver *driver;
2430
2431         TAILQ_FOREACH(driver, &cryptodev_driver_list, next)
2432                 if (driver->id == driver_id)
2433                         return driver->driver->name;
2434         return NULL;
2435 }
2436
2437 uint8_t
2438 rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv,
2439                 const struct rte_driver *drv)
2440 {
2441         crypto_drv->driver = drv;
2442         crypto_drv->id = nb_drivers;
2443
2444         TAILQ_INSERT_TAIL(&cryptodev_driver_list, crypto_drv, next);
2445
2446         return nb_drivers++;
2447 }
2448
2449 RTE_INIT(cryptodev_init_fp_ops)
2450 {
2451         uint32_t i;
2452
2453         for (i = 0; i != RTE_DIM(rte_crypto_fp_ops); i++)
2454                 cryptodev_fp_ops_reset(rte_crypto_fp_ops + i);
2455 }
2456
2457 static int
2458 cryptodev_handle_dev_list(const char *cmd __rte_unused,
2459                 const char *params __rte_unused,
2460                 struct rte_tel_data *d)
2461 {
2462         int dev_id;
2463
2464         if (rte_cryptodev_count() < 1)
2465                 return -EINVAL;
2466
2467         rte_tel_data_start_array(d, RTE_TEL_INT_VAL);
2468         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++)
2469                 if (rte_cryptodev_is_valid_dev(dev_id))
2470                         rte_tel_data_add_array_int(d, dev_id);
2471
2472         return 0;
2473 }
2474
2475 static int
2476 cryptodev_handle_dev_info(const char *cmd __rte_unused,
2477                 const char *params, struct rte_tel_data *d)
2478 {
2479         struct rte_cryptodev_info cryptodev_info;
2480         int dev_id;
2481         char *end_param;
2482
2483         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
2484                 return -EINVAL;
2485
2486         dev_id = strtoul(params, &end_param, 0);
2487         if (*end_param != '\0')
2488                 CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
2489         if (!rte_cryptodev_is_valid_dev(dev_id))
2490                 return -EINVAL;
2491
2492         rte_cryptodev_info_get(dev_id, &cryptodev_info);
2493
2494         rte_tel_data_start_dict(d);
2495         rte_tel_data_add_dict_string(d, "device_name",
2496                 cryptodev_info.device->name);
2497         rte_tel_data_add_dict_int(d, "max_nb_queue_pairs",
2498                 cryptodev_info.max_nb_queue_pairs);
2499
2500         return 0;
2501 }
2502
2503 #define ADD_DICT_STAT(s) rte_tel_data_add_dict_u64(d, #s, cryptodev_stats.s)
2504
2505 static int
2506 cryptodev_handle_dev_stats(const char *cmd __rte_unused,
2507                 const char *params,
2508                 struct rte_tel_data *d)
2509 {
2510         struct rte_cryptodev_stats cryptodev_stats;
2511         int dev_id, ret;
2512         char *end_param;
2513
2514         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
2515                 return -EINVAL;
2516
2517         dev_id = strtoul(params, &end_param, 0);
2518         if (*end_param != '\0')
2519                 CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
2520         if (!rte_cryptodev_is_valid_dev(dev_id))
2521                 return -EINVAL;
2522
2523         ret = rte_cryptodev_stats_get(dev_id, &cryptodev_stats);
2524         if (ret < 0)
2525                 return ret;
2526
2527         rte_tel_data_start_dict(d);
2528         ADD_DICT_STAT(enqueued_count);
2529         ADD_DICT_STAT(dequeued_count);
2530         ADD_DICT_STAT(enqueue_err_count);
2531         ADD_DICT_STAT(dequeue_err_count);
2532
2533         return 0;
2534 }
2535
2536 #define CRYPTO_CAPS_SZ                                             \
2537         (RTE_ALIGN_CEIL(sizeof(struct rte_cryptodev_capabilities), \
2538                                         sizeof(uint64_t)) /        \
2539          sizeof(uint64_t))
2540
2541 static int
2542 crypto_caps_array(struct rte_tel_data *d,
2543                   const struct rte_cryptodev_capabilities *capabilities)
2544 {
2545         const struct rte_cryptodev_capabilities *dev_caps;
2546         uint64_t caps_val[CRYPTO_CAPS_SZ];
2547         unsigned int i = 0, j;
2548
2549         rte_tel_data_start_array(d, RTE_TEL_U64_VAL);
2550
2551         while ((dev_caps = &capabilities[i++])->op !=
2552                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
2553                 memset(&caps_val, 0, CRYPTO_CAPS_SZ * sizeof(caps_val[0]));
2554                 rte_memcpy(caps_val, dev_caps, sizeof(capabilities[0]));
2555                 for (j = 0; j < CRYPTO_CAPS_SZ; j++)
2556                         rte_tel_data_add_array_u64(d, caps_val[j]);
2557         }
2558
2559         return i;
2560 }
2561
2562 static int
2563 cryptodev_handle_dev_caps(const char *cmd __rte_unused, const char *params,
2564                           struct rte_tel_data *d)
2565 {
2566         struct rte_cryptodev_info dev_info;
2567         struct rte_tel_data *crypto_caps;
2568         int crypto_caps_n;
2569         char *end_param;
2570         int dev_id;
2571
2572         if (!params || strlen(params) == 0 || !isdigit(*params))
2573                 return -EINVAL;
2574
2575         dev_id = strtoul(params, &end_param, 0);
2576         if (*end_param != '\0')
2577                 CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
2578         if (!rte_cryptodev_is_valid_dev(dev_id))
2579                 return -EINVAL;
2580
2581         rte_tel_data_start_dict(d);
2582         crypto_caps = rte_tel_data_alloc();
2583         if (!crypto_caps)
2584                 return -ENOMEM;
2585
2586         rte_cryptodev_info_get(dev_id, &dev_info);
2587         crypto_caps_n = crypto_caps_array(crypto_caps, dev_info.capabilities);
2588         rte_tel_data_add_dict_container(d, "crypto_caps", crypto_caps, 0);
2589         rte_tel_data_add_dict_int(d, "crypto_caps_n", crypto_caps_n);
2590
2591         return 0;
2592 }
2593
2594 RTE_INIT(cryptodev_init_telemetry)
2595 {
2596         rte_telemetry_register_cmd("/cryptodev/info", cryptodev_handle_dev_info,
2597                         "Returns information for a cryptodev. Parameters: int dev_id");
2598         rte_telemetry_register_cmd("/cryptodev/list",
2599                         cryptodev_handle_dev_list,
2600                         "Returns list of available crypto devices by IDs. No parameters.");
2601         rte_telemetry_register_cmd("/cryptodev/stats",
2602                         cryptodev_handle_dev_stats,
2603                         "Returns the stats for a cryptodev. Parameters: int dev_id");
2604         rte_telemetry_register_cmd("/cryptodev/caps",
2605                         cryptodev_handle_dev_caps,
2606                         "Returns the capabilities for a cryptodev. Parameters: int dev_id");
2607 }