lib/cryptodev/rte_cryptodev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2020 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdint.h>
  14 #include <inttypes.h>
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_log.h>
  18 #include <rte_debug.h>
  19 #include <rte_dev.h>
  20 #include <rte_interrupts.h>
  21 #include <rte_memory.h>
  22 #include <rte_memcpy.h>
  23 #include <rte_memzone.h>
  24 #include <rte_launch.h>
  25 #include <rte_tailq.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_common.h>
  32 #include <rte_mempool.h>
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_errno.h>
  36 #include <rte_spinlock.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_telemetry.h>
  39
  40 #include "rte_crypto.h"
  41 #include "rte_cryptodev.h"
  42 #include "cryptodev_pmd.h"
  43 #include "rte_cryptodev_trace.h"
  44
  45 static uint8_t nb_drivers;
  46
  47 static struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
  48
  49 struct rte_cryptodev *rte_cryptodevs = rte_crypto_devices;
  50
  51 static struct rte_cryptodev_global cryptodev_globals = {
  52                 .devs                   = rte_crypto_devices,
  53                 .data                   = { NULL },
  54                 .nb_devs                = 0
  55 };
  56
  57 /* Public fastpath APIs. */
  58 struct rte_crypto_fp_ops rte_crypto_fp_ops[RTE_CRYPTO_MAX_DEVS];
  59
  60 /* spinlock for crypto device callbacks */
  61 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  62
  63 /**
  64  * The user application callback description.
  65  *
  66  * It contains callback address to be registered by user application,
  67  * the pointer to the parameters for callback, and the event type.
  68  */
  69 struct rte_cryptodev_callback {
  70         TAILQ_ENTRY(rte_cryptodev_callback) next; /**< Callbacks list */
  71         rte_cryptodev_cb_fn cb_fn;              /**< Callback address */
  72         void *cb_arg;                           /**< Parameter for callback */
  73         enum rte_cryptodev_event_type event;    /**< Interrupt event type */
  74         uint32_t active;                        /**< Callback is executing */
  75 };
  76
  77 /**
  78  * The crypto cipher algorithm strings identifiers.
  79  * It could be used in application command line.
  80  */
  81 const char *
  82 rte_crypto_cipher_algorithm_strings[] = {
  83         [RTE_CRYPTO_CIPHER_3DES_CBC]    = "3des-cbc",
  84         [RTE_CRYPTO_CIPHER_3DES_ECB]    = "3des-ecb",
  85         [RTE_CRYPTO_CIPHER_3DES_CTR]    = "3des-ctr",
  86
  87         [RTE_CRYPTO_CIPHER_AES_CBC]     = "aes-cbc",
  88         [RTE_CRYPTO_CIPHER_AES_CTR]     = "aes-ctr",
  89         [RTE_CRYPTO_CIPHER_AES_DOCSISBPI]       = "aes-docsisbpi",
  90         [RTE_CRYPTO_CIPHER_AES_ECB]     = "aes-ecb",
  91         [RTE_CRYPTO_CIPHER_AES_F8]      = "aes-f8",
  92         [RTE_CRYPTO_CIPHER_AES_XTS]     = "aes-xts",
  93
  94         [RTE_CRYPTO_CIPHER_ARC4]        = "arc4",
  95
  96         [RTE_CRYPTO_CIPHER_DES_CBC]     = "des-cbc",
  97         [RTE_CRYPTO_CIPHER_DES_DOCSISBPI]       = "des-docsisbpi",
  98
  99         [RTE_CRYPTO_CIPHER_NULL]        = "null",
 100
 101         [RTE_CRYPTO_CIPHER_KASUMI_F8]   = "kasumi-f8",
 102         [RTE_CRYPTO_CIPHER_SNOW3G_UEA2] = "snow3g-uea2",
 103         [RTE_CRYPTO_CIPHER_ZUC_EEA3]    = "zuc-eea3"
 104 };
 105
 106 /**
 107  * The crypto cipher operation strings identifiers.
 108  * It could be used in application command line.
 109  */
 110 const char *
 111 rte_crypto_cipher_operation_strings[] = {
 112                 [RTE_CRYPTO_CIPHER_OP_ENCRYPT]  = "encrypt",
 113                 [RTE_CRYPTO_CIPHER_OP_DECRYPT]  = "decrypt"
 114 };
 115
 116 /**
 117  * The crypto auth algorithm strings identifiers.
 118  * It could be used in application command line.
 119  */
 120 const char *
 121 rte_crypto_auth_algorithm_strings[] = {
 122         [RTE_CRYPTO_AUTH_AES_CBC_MAC]   = "aes-cbc-mac",
 123         [RTE_CRYPTO_AUTH_AES_CMAC]      = "aes-cmac",
 124         [RTE_CRYPTO_AUTH_AES_GMAC]      = "aes-gmac",
 125         [RTE_CRYPTO_AUTH_AES_XCBC_MAC]  = "aes-xcbc-mac",
 126
 127         [RTE_CRYPTO_AUTH_MD5]           = "md5",
 128         [RTE_CRYPTO_AUTH_MD5_HMAC]      = "md5-hmac",
 129
 130         [RTE_CRYPTO_AUTH_NULL]          = "null",
 131
 132         [RTE_CRYPTO_AUTH_SHA1]          = "sha1",
 133         [RTE_CRYPTO_AUTH_SHA1_HMAC]     = "sha1-hmac",
 134
 135         [RTE_CRYPTO_AUTH_SHA224]        = "sha2-224",
 136         [RTE_CRYPTO_AUTH_SHA224_HMAC]   = "sha2-224-hmac",
 137         [RTE_CRYPTO_AUTH_SHA256]        = "sha2-256",
 138         [RTE_CRYPTO_AUTH_SHA256_HMAC]   = "sha2-256-hmac",
 139         [RTE_CRYPTO_AUTH_SHA384]        = "sha2-384",
 140         [RTE_CRYPTO_AUTH_SHA384_HMAC]   = "sha2-384-hmac",
 141         [RTE_CRYPTO_AUTH_SHA512]        = "sha2-512",
 142         [RTE_CRYPTO_AUTH_SHA512_HMAC]   = "sha2-512-hmac",
 143
 144         [RTE_CRYPTO_AUTH_KASUMI_F9]     = "kasumi-f9",
 145         [RTE_CRYPTO_AUTH_SNOW3G_UIA2]   = "snow3g-uia2",
 146         [RTE_CRYPTO_AUTH_ZUC_EIA3]      = "zuc-eia3"
 147 };
 148
 149 /**
 150  * The crypto AEAD algorithm strings identifiers.
 151  * It could be used in application command line.
 152  */
 153 const char *
 154 rte_crypto_aead_algorithm_strings[] = {
 155         [RTE_CRYPTO_AEAD_AES_CCM]       = "aes-ccm",
 156         [RTE_CRYPTO_AEAD_AES_GCM]       = "aes-gcm",
 157         [RTE_CRYPTO_AEAD_CHACHA20_POLY1305] = "chacha20-poly1305"
 158 };
 159
 160 /**
 161  * The crypto AEAD operation strings identifiers.
 162  * It could be used in application command line.
 163  */
 164 const char *
 165 rte_crypto_aead_operation_strings[] = {
 166         [RTE_CRYPTO_AEAD_OP_ENCRYPT]    = "encrypt",
 167         [RTE_CRYPTO_AEAD_OP_DECRYPT]    = "decrypt"
 168 };
 169
 170 /**
 171  * Asymmetric crypto transform operation strings identifiers.
 172  */
 173 const char *rte_crypto_asym_xform_strings[] = {
 174         [RTE_CRYPTO_ASYM_XFORM_NONE]    = "none",
 175         [RTE_CRYPTO_ASYM_XFORM_RSA]     = "rsa",
 176         [RTE_CRYPTO_ASYM_XFORM_MODEX]   = "modexp",
 177         [RTE_CRYPTO_ASYM_XFORM_MODINV]  = "modinv",
 178         [RTE_CRYPTO_ASYM_XFORM_DH]      = "dh",
 179         [RTE_CRYPTO_ASYM_XFORM_DSA]     = "dsa",
 180         [RTE_CRYPTO_ASYM_XFORM_ECDSA]   = "ecdsa",
 181         [RTE_CRYPTO_ASYM_XFORM_ECPM]    = "ecpm",
 182 };
 183
 184 /**
 185  * Asymmetric crypto operation strings identifiers.
 186  */
 187 const char *rte_crypto_asym_op_strings[] = {
 188         [RTE_CRYPTO_ASYM_OP_ENCRYPT]    = "encrypt",
 189         [RTE_CRYPTO_ASYM_OP_DECRYPT]    = "decrypt",
 190         [RTE_CRYPTO_ASYM_OP_SIGN]       = "sign",
 191         [RTE_CRYPTO_ASYM_OP_VERIFY]     = "verify",
 192         [RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE]       = "priv_key_generate",
 193         [RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE] = "pub_key_generate",
 194         [RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE] = "sharedsecret_compute",
 195 };
 196
 197 /**
 198  * The private data structure stored in the sym session mempool private data.
 199  */
 200 struct rte_cryptodev_sym_session_pool_private_data {
 201         uint16_t nb_drivers;
 202         /**< number of elements in sess_data array */
 203         uint16_t user_data_sz;
 204         /**< session user data will be placed after sess_data */
 205 };
 206
 207 /**
 208  * The private data structure stored in the asym session mempool private data.
 209  */
 210 struct rte_cryptodev_asym_session_pool_private_data {
 211         uint16_t max_priv_session_sz;
 212         /**< Size of private session data used when creating mempool */
 213         uint16_t user_data_sz;
 214         /**< Session user data will be placed after sess_private_data */
 215 };
 216
 217 int
 218 rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum,
 219                 const char *algo_string)
 220 {
 221         unsigned int i;
 222
 223         for (i = 1; i < RTE_DIM(rte_crypto_cipher_algorithm_strings); i++) {
 224                 if (strcmp(algo_string, rte_crypto_cipher_algorithm_strings[i]) == 0) {
 225                         *algo_enum = (enum rte_crypto_cipher_algorithm) i;
 226                         return 0;
 227                 }
 228         }
 229
 230         /* Invalid string */
 231         return -1;
 232 }
 233
 234 int
 235 rte_cryptodev_get_auth_algo_enum(enum rte_crypto_auth_algorithm *algo_enum,
 236                 const char *algo_string)
 237 {
 238         unsigned int i;
 239
 240         for (i = 1; i < RTE_DIM(rte_crypto_auth_algorithm_strings); i++) {
 241                 if (strcmp(algo_string, rte_crypto_auth_algorithm_strings[i]) == 0) {
 242                         *algo_enum = (enum rte_crypto_auth_algorithm) i;
 243                         return 0;
 244                 }
 245         }
 246
 247         /* Invalid string */
 248         return -1;
 249 }
 250
 251 int
 252 rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum,
 253                 const char *algo_string)
 254 {
 255         unsigned int i;
 256
 257         for (i = 1; i < RTE_DIM(rte_crypto_aead_algorithm_strings); i++) {
 258                 if (strcmp(algo_string, rte_crypto_aead_algorithm_strings[i]) == 0) {
 259                         *algo_enum = (enum rte_crypto_aead_algorithm) i;
 260                         return 0;
 261                 }
 262         }
 263
 264         /* Invalid string */
 265         return -1;
 266 }
 267
 268 int
 269 rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
 270                 const char *xform_string)
 271 {
 272         unsigned int i;
 273
 274         for (i = 1; i < RTE_DIM(rte_crypto_asym_xform_strings); i++) {
 275                 if (strcmp(xform_string,
 276                         rte_crypto_asym_xform_strings[i]) == 0) {
 277                         *xform_enum = (enum rte_crypto_asym_xform_type) i;
 278                         return 0;
 279                 }
 280         }
 281
 282         /* Invalid string */
 283         return -1;
 284 }
 285
 286 /**
 287  * The crypto auth operation strings identifiers.
 288  * It could be used in application command line.
 289  */
 290 const char *
 291 rte_crypto_auth_operation_strings[] = {
 292                 [RTE_CRYPTO_AUTH_OP_VERIFY]     = "verify",
 293                 [RTE_CRYPTO_AUTH_OP_GENERATE]   = "generate"
 294 };
 295
 296 const struct rte_cryptodev_symmetric_capability *
 297 rte_cryptodev_sym_capability_get(uint8_t dev_id,
 298                 const struct rte_cryptodev_sym_capability_idx *idx)
 299 {
 300         const struct rte_cryptodev_capabilities *capability;
 301         struct rte_cryptodev_info dev_info;
 302         int i = 0;
 303
 304         rte_cryptodev_info_get(dev_id, &dev_info);
 305
 306         while ((capability = &dev_info.capabilities[i++])->op !=
 307                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 308                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
 309                         continue;
 310
 311                 if (capability->sym.xform_type != idx->type)
 312                         continue;
 313
 314                 if (idx->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 315                         capability->sym.auth.algo == idx->algo.auth)
 316                         return &capability->sym;
 317
 318                 if (idx->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 319                         capability->sym.cipher.algo == idx->algo.cipher)
 320                         return &capability->sym;
 321
 322                 if (idx->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
 323                                 capability->sym.aead.algo == idx->algo.aead)
 324                         return &capability->sym;
 325         }
 326
 327         return NULL;
 328 }
 329
 330 static int
 331 param_range_check(uint16_t size, const struct rte_crypto_param_range *range)
 332 {
 333         unsigned int next_size;
 334
 335         /* Check lower/upper bounds */
 336         if (size < range->min)
 337                 return -1;
 338
 339         if (size > range->max)
 340                 return -1;
 341
 342         /* If range is actually only one value, size is correct */
 343         if (range->increment == 0)
 344                 return 0;
 345
 346         /* Check if value is one of the supported sizes */
 347         for (next_size = range->min; next_size <= range->max;
 348                         next_size += range->increment)
 349                 if (size == next_size)
 350                         return 0;
 351
 352         return -1;
 353 }
 354
 355 const struct rte_cryptodev_asymmetric_xform_capability *
 356 rte_cryptodev_asym_capability_get(uint8_t dev_id,
 357                 const struct rte_cryptodev_asym_capability_idx *idx)
 358 {
 359         const struct rte_cryptodev_capabilities *capability;
 360         struct rte_cryptodev_info dev_info;
 361         unsigned int i = 0;
 362
 363         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 364         rte_cryptodev_info_get(dev_id, &dev_info);
 365
 366         while ((capability = &dev_info.capabilities[i++])->op !=
 367                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 368                 if (capability->op != RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
 369                         continue;
 370
 371                 if (capability->asym.xform_capa.xform_type == idx->type)
 372                         return &capability->asym.xform_capa;
 373         }
 374         return NULL;
 375 };
 376
 377 int
 378 rte_cryptodev_sym_capability_check_cipher(
 379                 const struct rte_cryptodev_symmetric_capability *capability,
 380                 uint16_t key_size, uint16_t iv_size)
 381 {
 382         if (param_range_check(key_size, &capability->cipher.key_size) != 0)
 383                 return -1;
 384
 385         if (param_range_check(iv_size, &capability->cipher.iv_size) != 0)
 386                 return -1;
 387
 388         return 0;
 389 }
 390
 391 int
 392 rte_cryptodev_sym_capability_check_auth(
 393                 const struct rte_cryptodev_symmetric_capability *capability,
 394                 uint16_t key_size, uint16_t digest_size, uint16_t iv_size)
 395 {
 396         if (param_range_check(key_size, &capability->auth.key_size) != 0)
 397                 return -1;
 398
 399         if (param_range_check(digest_size, &capability->auth.digest_size) != 0)
 400                 return -1;
 401
 402         if (param_range_check(iv_size, &capability->auth.iv_size) != 0)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 int
 409 rte_cryptodev_sym_capability_check_aead(
 410                 const struct rte_cryptodev_symmetric_capability *capability,
 411                 uint16_t key_size, uint16_t digest_size, uint16_t aad_size,
 412                 uint16_t iv_size)
 413 {
 414         if (param_range_check(key_size, &capability->aead.key_size) != 0)
 415                 return -1;
 416
 417         if (param_range_check(digest_size, &capability->aead.digest_size) != 0)
 418                 return -1;
 419
 420         if (param_range_check(aad_size, &capability->aead.aad_size) != 0)
 421                 return -1;
 422
 423         if (param_range_check(iv_size, &capability->aead.iv_size) != 0)
 424                 return -1;
 425
 426         return 0;
 427 }
 428 int
 429 rte_cryptodev_asym_xform_capability_check_optype(
 430         const struct rte_cryptodev_asymmetric_xform_capability *capability,
 431         enum rte_crypto_asym_op_type op_type)
 432 {
 433         if (capability->op_types & (1 << op_type))
 434                 return 1;
 435
 436         return 0;
 437 }
 438
 439 int
 440 rte_cryptodev_asym_xform_capability_check_modlen(
 441         const struct rte_cryptodev_asymmetric_xform_capability *capability,
 442         uint16_t modlen)
 443 {
 444         /* no need to check for limits, if min or max = 0 */
 445         if (capability->modlen.min != 0) {
 446                 if (modlen < capability->modlen.min)
 447                         return -1;
 448         }
 449
 450         if (capability->modlen.max != 0) {
 451                 if (modlen > capability->modlen.max)
 452                         return -1;
 453         }
 454
 455         /* in any case, check if given modlen is module increment */
 456         if (capability->modlen.increment != 0) {
 457                 if (modlen % (capability->modlen.increment))
 458                         return -1;
 459         }
 460
 461         return 0;
 462 }
 463
 464 /* spinlock for crypto device enq callbacks */
 465 static rte_spinlock_t rte_cryptodev_callback_lock = RTE_SPINLOCK_INITIALIZER;
 466
 467 static void
 468 cryptodev_cb_cleanup(struct rte_cryptodev *dev)
 469 {
 470         struct rte_cryptodev_cb_rcu *list;
 471         struct rte_cryptodev_cb *cb, *next;
 472         uint16_t qp_id;
 473
 474         if (dev->enq_cbs == NULL && dev->deq_cbs == NULL)
 475                 return;
 476
 477         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 478                 list = &dev->enq_cbs[qp_id];
 479                 cb = list->next;
 480                 while (cb != NULL) {
 481                         next = cb->next;
 482                         rte_free(cb);
 483                         cb = next;
 484                 }
 485
 486                 rte_free(list->qsbr);
 487         }
 488
 489         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 490                 list = &dev->deq_cbs[qp_id];
 491                 cb = list->next;
 492                 while (cb != NULL) {
 493                         next = cb->next;
 494                         rte_free(cb);
 495                         cb = next;
 496                 }
 497
 498                 rte_free(list->qsbr);
 499         }
 500
 501         rte_free(dev->enq_cbs);
 502         dev->enq_cbs = NULL;
 503         rte_free(dev->deq_cbs);
 504         dev->deq_cbs = NULL;
 505 }
 506
 507 static int
 508 cryptodev_cb_init(struct rte_cryptodev *dev)
 509 {
 510         struct rte_cryptodev_cb_rcu *list;
 511         struct rte_rcu_qsbr *qsbr;
 512         uint16_t qp_id;
 513         size_t size;
 514
 515         /* Max thread set to 1, as one DP thread accessing a queue-pair */
 516         const uint32_t max_threads = 1;
 517
 518         dev->enq_cbs = rte_zmalloc(NULL,
 519                                    sizeof(struct rte_cryptodev_cb_rcu) *
 520                                    dev->data->nb_queue_pairs, 0);
 521         if (dev->enq_cbs == NULL) {
 522                 CDEV_LOG_ERR("Failed to allocate memory for enq callbacks");
 523                 return -ENOMEM;
 524         }
 525
 526         dev->deq_cbs = rte_zmalloc(NULL,
 527                                    sizeof(struct rte_cryptodev_cb_rcu) *
 528                                    dev->data->nb_queue_pairs, 0);
 529         if (dev->deq_cbs == NULL) {
 530                 CDEV_LOG_ERR("Failed to allocate memory for deq callbacks");
 531                 rte_free(dev->enq_cbs);
 532                 return -ENOMEM;
 533         }
 534
 535         /* Create RCU QSBR variable */
 536         size = rte_rcu_qsbr_get_memsize(max_threads);
 537
 538         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 539                 list = &dev->enq_cbs[qp_id];
 540                 qsbr = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 541                 if (qsbr == NULL) {
 542                         CDEV_LOG_ERR("Failed to allocate memory for RCU on "
 543                                 "queue_pair_id=%d", qp_id);
 544                         goto cb_init_err;
 545                 }
 546
 547                 if (rte_rcu_qsbr_init(qsbr, max_threads)) {
 548                         CDEV_LOG_ERR("Failed to initialize for RCU on "
 549                                 "queue_pair_id=%d", qp_id);
 550                         goto cb_init_err;
 551                 }
 552
 553                 list->qsbr = qsbr;
 554         }
 555
 556         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 557                 list = &dev->deq_cbs[qp_id];
 558                 qsbr = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 559                 if (qsbr == NULL) {
 560                         CDEV_LOG_ERR("Failed to allocate memory for RCU on "
 561                                 "queue_pair_id=%d", qp_id);
 562                         goto cb_init_err;
 563                 }
 564
 565                 if (rte_rcu_qsbr_init(qsbr, max_threads)) {
 566                         CDEV_LOG_ERR("Failed to initialize for RCU on "
 567                                 "queue_pair_id=%d", qp_id);
 568                         goto cb_init_err;
 569                 }
 570
 571                 list->qsbr = qsbr;
 572         }
 573
 574         return 0;
 575
 576 cb_init_err:
 577         cryptodev_cb_cleanup(dev);
 578         return -ENOMEM;
 579 }
 580
 581 const char *
 582 rte_cryptodev_get_feature_name(uint64_t flag)
 583 {
 584         switch (flag) {
 585         case RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO:
 586                 return "SYMMETRIC_CRYPTO";
 587         case RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO:
 588                 return "ASYMMETRIC_CRYPTO";
 589         case RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING:
 590                 return "SYM_OPERATION_CHAINING";
 591         case RTE_CRYPTODEV_FF_CPU_SSE:
 592                 return "CPU_SSE";
 593         case RTE_CRYPTODEV_FF_CPU_AVX:
 594                 return "CPU_AVX";
 595         case RTE_CRYPTODEV_FF_CPU_AVX2:
 596                 return "CPU_AVX2";
 597         case RTE_CRYPTODEV_FF_CPU_AVX512:
 598                 return "CPU_AVX512";
 599         case RTE_CRYPTODEV_FF_CPU_AESNI:
 600                 return "CPU_AESNI";
 601         case RTE_CRYPTODEV_FF_HW_ACCELERATED:
 602                 return "HW_ACCELERATED";
 603         case RTE_CRYPTODEV_FF_IN_PLACE_SGL:
 604                 return "IN_PLACE_SGL";
 605         case RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT:
 606                 return "OOP_SGL_IN_SGL_OUT";
 607         case RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT:
 608                 return "OOP_SGL_IN_LB_OUT";
 609         case RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT:
 610                 return "OOP_LB_IN_SGL_OUT";
 611         case RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT:
 612                 return "OOP_LB_IN_LB_OUT";
 613         case RTE_CRYPTODEV_FF_CPU_NEON:
 614                 return "CPU_NEON";
 615         case RTE_CRYPTODEV_FF_CPU_ARM_CE:
 616                 return "CPU_ARM_CE";
 617         case RTE_CRYPTODEV_FF_SECURITY:
 618                 return "SECURITY_PROTOCOL";
 619         case RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP:
 620                 return "RSA_PRIV_OP_KEY_EXP";
 621         case RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT:
 622                 return "RSA_PRIV_OP_KEY_QT";
 623         case RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED:
 624                 return "DIGEST_ENCRYPTED";
 625         case RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO:
 626                 return "SYM_CPU_CRYPTO";
 627         case RTE_CRYPTODEV_FF_ASYM_SESSIONLESS:
 628                 return "ASYM_SESSIONLESS";
 629         case RTE_CRYPTODEV_FF_SYM_SESSIONLESS:
 630                 return "SYM_SESSIONLESS";
 631         case RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA:
 632                 return "NON_BYTE_ALIGNED_DATA";
 633         case RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS:
 634                 return "CIPHER_MULTIPLE_DATA_UNITS";
 635         case RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY:
 636                 return "CIPHER_WRAPPED_KEY";
 637         default:
 638                 return NULL;
 639         }
 640 }
 641
 642 struct rte_cryptodev *
 643 rte_cryptodev_pmd_get_dev(uint8_t dev_id)
 644 {
 645         return &cryptodev_globals.devs[dev_id];
 646 }
 647
 648 struct rte_cryptodev *
 649 rte_cryptodev_pmd_get_named_dev(const char *name)
 650 {
 651         struct rte_cryptodev *dev;
 652         unsigned int i;
 653
 654         if (name == NULL)
 655                 return NULL;
 656
 657         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
 658                 dev = &cryptodev_globals.devs[i];
 659
 660                 if ((dev->attached == RTE_CRYPTODEV_ATTACHED) &&
 661                                 (strcmp(dev->data->name, name) == 0))
 662                         return dev;
 663         }
 664
 665         return NULL;
 666 }
 667
 668 static inline uint8_t
 669 rte_cryptodev_is_valid_device_data(uint8_t dev_id)
 670 {
 671         if (dev_id >= RTE_CRYPTO_MAX_DEVS ||
 672                         rte_crypto_devices[dev_id].data == NULL)
 673                 return 0;
 674
 675         return 1;
 676 }
 677
 678 unsigned int
 679 rte_cryptodev_is_valid_dev(uint8_t dev_id)
 680 {
 681         struct rte_cryptodev *dev = NULL;
 682
 683         if (!rte_cryptodev_is_valid_device_data(dev_id))
 684                 return 0;
 685
 686         dev = rte_cryptodev_pmd_get_dev(dev_id);
 687         if (dev->attached != RTE_CRYPTODEV_ATTACHED)
 688                 return 0;
 689         else
 690                 return 1;
 691 }
 692
 693
 694 int
 695 rte_cryptodev_get_dev_id(const char *name)
 696 {
 697         unsigned i;
 698
 699         if (name == NULL)
 700                 return -1;
 701
 702         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
 703                 if (!rte_cryptodev_is_valid_device_data(i))
 704                         continue;
 705                 if ((strcmp(cryptodev_globals.devs[i].data->name, name)
 706                                 == 0) &&
 707                                 (cryptodev_globals.devs[i].attached ==
 708                                                 RTE_CRYPTODEV_ATTACHED))
 709                         return i;
 710         }
 711
 712         return -1;
 713 }
 714
 715 uint8_t
 716 rte_cryptodev_count(void)
 717 {
 718         return cryptodev_globals.nb_devs;
 719 }
 720
 721 uint8_t
 722 rte_cryptodev_device_count_by_driver(uint8_t driver_id)
 723 {
 724         uint8_t i, dev_count = 0;
 725
 726         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++)
 727                 if (cryptodev_globals.devs[i].driver_id == driver_id &&
 728                         cryptodev_globals.devs[i].attached ==
 729                                         RTE_CRYPTODEV_ATTACHED)
 730                         dev_count++;
 731
 732         return dev_count;
 733 }
 734
 735 uint8_t
 736 rte_cryptodev_devices_get(const char *driver_name, uint8_t *devices,
 737         uint8_t nb_devices)
 738 {
 739         uint8_t i, count = 0;
 740         struct rte_cryptodev *devs = cryptodev_globals.devs;
 741
 742         for (i = 0; i < RTE_CRYPTO_MAX_DEVS && count < nb_devices; i++) {
 743                 if (!rte_cryptodev_is_valid_device_data(i))
 744                         continue;
 745
 746                 if (devs[i].attached == RTE_CRYPTODEV_ATTACHED) {
 747                         int cmp;
 748
 749                         cmp = strncmp(devs[i].device->driver->name,
 750                                         driver_name,
 751                                         strlen(driver_name) + 1);
 752
 753                         if (cmp == 0)
 754                                 devices[count++] = devs[i].data->dev_id;
 755                 }
 756         }
 757
 758         return count;
 759 }
 760
 761 void *
 762 rte_cryptodev_get_sec_ctx(uint8_t dev_id)
 763 {
 764         if (dev_id < RTE_CRYPTO_MAX_DEVS &&
 765                         (rte_crypto_devices[dev_id].feature_flags &
 766                         RTE_CRYPTODEV_FF_SECURITY))
 767                 return rte_crypto_devices[dev_id].security_ctx;
 768
 769         return NULL;
 770 }
 771
 772 int
 773 rte_cryptodev_socket_id(uint8_t dev_id)
 774 {
 775         struct rte_cryptodev *dev;
 776
 777         if (!rte_cryptodev_is_valid_dev(dev_id))
 778                 return -1;
 779
 780         dev = rte_cryptodev_pmd_get_dev(dev_id);
 781
 782         return dev->data->socket_id;
 783 }
 784
 785 static inline int
 786 rte_cryptodev_data_alloc(uint8_t dev_id, struct rte_cryptodev_data **data,
 787                 int socket_id)
 788 {
 789         char mz_name[RTE_MEMZONE_NAMESIZE];
 790         const struct rte_memzone *mz;
 791         int n;
 792
 793         /* generate memzone name */
 794         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 795         if (n >= (int)sizeof(mz_name))
 796                 return -EINVAL;
 797
 798         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 799                 mz = rte_memzone_reserve(mz_name,
 800                                 sizeof(struct rte_cryptodev_data),
 801                                 socket_id, 0);
 802                 CDEV_LOG_DEBUG("PRIMARY:reserved memzone for %s (%p)",
 803                                 mz_name, mz);
 804         } else {
 805                 mz = rte_memzone_lookup(mz_name);
 806                 CDEV_LOG_DEBUG("SECONDARY:looked up memzone for %s (%p)",
 807                                 mz_name, mz);
 808         }
 809
 810         if (mz == NULL)
 811                 return -ENOMEM;
 812
 813         *data = mz->addr;
 814         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 815                 memset(*data, 0, sizeof(struct rte_cryptodev_data));
 816
 817         return 0;
 818 }
 819
 820 static inline int
 821 rte_cryptodev_data_free(uint8_t dev_id, struct rte_cryptodev_data **data)
 822 {
 823         char mz_name[RTE_MEMZONE_NAMESIZE];
 824         const struct rte_memzone *mz;
 825         int n;
 826
 827         /* generate memzone name */
 828         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 829         if (n >= (int)sizeof(mz_name))
 830                 return -EINVAL;
 831
 832         mz = rte_memzone_lookup(mz_name);
 833         if (mz == NULL)
 834                 return -ENOMEM;
 835
 836         RTE_ASSERT(*data == mz->addr);
 837         *data = NULL;
 838
 839         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 840                 CDEV_LOG_DEBUG("PRIMARY:free memzone of %s (%p)",
 841                                 mz_name, mz);
 842                 return rte_memzone_free(mz);
 843         } else {
 844                 CDEV_LOG_DEBUG("SECONDARY:don't free memzone of %s (%p)",
 845                                 mz_name, mz);
 846         }
 847
 848         return 0;
 849 }
 850
 851 static uint8_t
 852 rte_cryptodev_find_free_device_index(void)
 853 {
 854         uint8_t dev_id;
 855
 856         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++) {
 857                 if (rte_crypto_devices[dev_id].attached ==
 858                                 RTE_CRYPTODEV_DETACHED)
 859                         return dev_id;
 860         }
 861         return RTE_CRYPTO_MAX_DEVS;
 862 }
 863
 864 struct rte_cryptodev *
 865 rte_cryptodev_pmd_allocate(const char *name, int socket_id)
 866 {
 867         struct rte_cryptodev *cryptodev;
 868         uint8_t dev_id;
 869
 870         if (rte_cryptodev_pmd_get_named_dev(name) != NULL) {
 871                 CDEV_LOG_ERR("Crypto device with name %s already "
 872                                 "allocated!", name);
 873                 return NULL;
 874         }
 875
 876         dev_id = rte_cryptodev_find_free_device_index();
 877         if (dev_id == RTE_CRYPTO_MAX_DEVS) {
 878                 CDEV_LOG_ERR("Reached maximum number of crypto devices");
 879                 return NULL;
 880         }
 881
 882         cryptodev = rte_cryptodev_pmd_get_dev(dev_id);
 883
 884         if (cryptodev->data == NULL) {
 885                 struct rte_cryptodev_data **cryptodev_data =
 886                                 &cryptodev_globals.data[dev_id];
 887
 888                 int retval = rte_cryptodev_data_alloc(dev_id, cryptodev_data,
 889                                 socket_id);
 890
 891                 if (retval < 0 || *cryptodev_data == NULL)
 892                         return NULL;
 893
 894                 cryptodev->data = *cryptodev_data;
 895
 896                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 897                         strlcpy(cryptodev->data->name, name,
 898                                 RTE_CRYPTODEV_NAME_MAX_LEN);
 899
 900                         cryptodev->data->dev_id = dev_id;
 901                         cryptodev->data->socket_id = socket_id;
 902                         cryptodev->data->dev_started = 0;
 903                         CDEV_LOG_DEBUG("PRIMARY:init data");
 904                 }
 905
 906                 CDEV_LOG_DEBUG("Data for %s: dev_id %d, socket %d, started %d",
 907                                 cryptodev->data->name,
 908                                 cryptodev->data->dev_id,
 909                                 cryptodev->data->socket_id,
 910                                 cryptodev->data->dev_started);
 911
 912                 /* init user callbacks */
 913                 TAILQ_INIT(&(cryptodev->link_intr_cbs));
 914
 915                 cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 916
 917                 cryptodev_globals.nb_devs++;
 918         }
 919
 920         return cryptodev;
 921 }
 922
 923 int
 924 rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 925 {
 926         int ret;
 927         uint8_t dev_id;
 928
 929         if (cryptodev == NULL)
 930                 return -EINVAL;
 931
 932         dev_id = cryptodev->data->dev_id;
 933
 934         cryptodev_fp_ops_reset(rte_crypto_fp_ops + dev_id);
 935
 936         /* Close device only if device operations have been set */
 937         if (cryptodev->dev_ops) {
 938                 ret = rte_cryptodev_close(dev_id);
 939                 if (ret < 0)
 940                         return ret;
 941         }
 942
 943         ret = rte_cryptodev_data_free(dev_id, &cryptodev_globals.data[dev_id]);
 944         if (ret < 0)
 945                 return ret;
 946
 947         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 948         cryptodev_globals.nb_devs--;
 949         return 0;
 950 }
 951
 952 uint16_t
 953 rte_cryptodev_queue_pair_count(uint8_t dev_id)
 954 {
 955         struct rte_cryptodev *dev;
 956
 957         if (!rte_cryptodev_is_valid_device_data(dev_id)) {
 958                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 959                 return 0;
 960         }
 961
 962         dev = &rte_crypto_devices[dev_id];
 963         return dev->data->nb_queue_pairs;
 964 }
 965
 966 static int
 967 rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 968                 int socket_id)
 969 {
 970         struct rte_cryptodev_info dev_info;
 971         void **qp;
 972         unsigned i;
 973
 974         if ((dev == NULL) || (nb_qpairs < 1)) {
 975                 CDEV_LOG_ERR("invalid param: dev %p, nb_queues %u",
 976                                                         dev, nb_qpairs);
 977                 return -EINVAL;
 978         }
 979
 980         CDEV_LOG_DEBUG("Setup %d queues pairs on device %u",
 981                         nb_qpairs, dev->data->dev_id);
 982
 983         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 984
 985         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 986         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 987
 988         if (nb_qpairs > (dev_info.max_nb_queue_pairs)) {
 989                 CDEV_LOG_ERR("Invalid num queue_pairs (%u) for dev %u",
 990                                 nb_qpairs, dev->data->dev_id);
 991             return -EINVAL;
 992         }
 993
 994         if (dev->data->queue_pairs == NULL) { /* first time configuration */
 995                 dev->data->queue_pairs = rte_zmalloc_socket(
 996                                 "cryptodev->queue_pairs",
 997                                 sizeof(dev->data->queue_pairs[0]) *
 998                                 dev_info.max_nb_queue_pairs,
 999                                 RTE_CACHE_LINE_SIZE, socket_id);
1000
1001                 if (dev->data->queue_pairs == NULL) {
1002                         dev->data->nb_queue_pairs = 0;
1003                         CDEV_LOG_ERR("failed to get memory for qp meta data, "
1004                                                         "nb_queues %u",
1005                                                         nb_qpairs);
1006                         return -(ENOMEM);
1007                 }
1008         } else { /* re-configure */
1009                 int ret;
1010                 uint16_t old_nb_queues = dev->data->nb_queue_pairs;
1011
1012                 qp = dev->data->queue_pairs;
1013
1014                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
1015                                 -ENOTSUP);
1016
1017                 for (i = nb_qpairs; i < old_nb_queues; i++) {
1018                         ret = (*dev->dev_ops->queue_pair_release)(dev, i);
1019                         if (ret < 0)
1020                                 return ret;
1021                         qp[i] = NULL;
1022                 }
1023
1024         }
1025         dev->data->nb_queue_pairs = nb_qpairs;
1026         return 0;
1027 }
1028
1029 int
1030 rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
1031 {
1032         struct rte_cryptodev *dev;
1033         int diag;
1034
1035         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1036                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1037                 return -EINVAL;
1038         }
1039
1040         dev = &rte_crypto_devices[dev_id];
1041
1042         if (dev->data->dev_started) {
1043                 CDEV_LOG_ERR(
1044                     "device %d must be stopped to allow configuration", dev_id);
1045                 return -EBUSY;
1046         }
1047
1048         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1049
1050         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1051         cryptodev_cb_cleanup(dev);
1052         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1053
1054         /* Setup new number of queue pairs and reconfigure device. */
1055         diag = rte_cryptodev_queue_pairs_config(dev, config->nb_queue_pairs,
1056                         config->socket_id);
1057         if (diag != 0) {
1058                 CDEV_LOG_ERR("dev%d rte_crypto_dev_queue_pairs_config = %d",
1059                                 dev_id, diag);
1060                 return diag;
1061         }
1062
1063         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1064         diag = cryptodev_cb_init(dev);
1065         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1066         if (diag) {
1067                 CDEV_LOG_ERR("Callback init failed for dev_id=%d", dev_id);
1068                 return diag;
1069         }
1070
1071         rte_cryptodev_trace_configure(dev_id, config);
1072         return (*dev->dev_ops->dev_configure)(dev, config);
1073 }
1074
1075 int
1076 rte_cryptodev_start(uint8_t dev_id)
1077 {
1078         struct rte_cryptodev *dev;
1079         int diag;
1080
1081         CDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id);
1082
1083         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1084                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1085                 return -EINVAL;
1086         }
1087
1088         dev = &rte_crypto_devices[dev_id];
1089
1090         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1091
1092         if (dev->data->dev_started != 0) {
1093                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already started",
1094                         dev_id);
1095                 return 0;
1096         }
1097
1098         diag = (*dev->dev_ops->dev_start)(dev);
1099         /* expose selection of PMD fast-path functions */
1100         cryptodev_fp_ops_set(rte_crypto_fp_ops + dev_id, dev);
1101
1102         rte_cryptodev_trace_start(dev_id, diag);
1103         if (diag == 0)
1104                 dev->data->dev_started = 1;
1105         else
1106                 return diag;
1107
1108         return 0;
1109 }
1110
1111 void
1112 rte_cryptodev_stop(uint8_t dev_id)
1113 {
1114         struct rte_cryptodev *dev;
1115
1116         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1117                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1118                 return;
1119         }
1120
1121         dev = &rte_crypto_devices[dev_id];
1122
1123         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1124
1125         if (dev->data->dev_started == 0) {
1126                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already stopped",
1127                         dev_id);
1128                 return;
1129         }
1130
1131         /* point fast-path functions to dummy ones */
1132         cryptodev_fp_ops_reset(rte_crypto_fp_ops + dev_id);
1133
1134         (*dev->dev_ops->dev_stop)(dev);
1135         rte_cryptodev_trace_stop(dev_id);
1136         dev->data->dev_started = 0;
1137 }
1138
1139 int
1140 rte_cryptodev_close(uint8_t dev_id)
1141 {
1142         struct rte_cryptodev *dev;
1143         int retval;
1144
1145         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1146                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1147                 return -1;
1148         }
1149
1150         dev = &rte_crypto_devices[dev_id];
1151
1152         /* Device must be stopped before it can be closed */
1153         if (dev->data->dev_started == 1) {
1154                 CDEV_LOG_ERR("Device %u must be stopped before closing",
1155                                 dev_id);
1156                 return -EBUSY;
1157         }
1158
1159         /* We can't close the device if there are outstanding sessions in use */
1160         if (dev->data->session_pool != NULL) {
1161                 if (!rte_mempool_full(dev->data->session_pool)) {
1162                         CDEV_LOG_ERR("dev_id=%u close failed, session mempool "
1163                                         "has sessions still in use, free "
1164                                         "all sessions before calling close",
1165                                         (unsigned)dev_id);
1166                         return -EBUSY;
1167                 }
1168         }
1169
1170         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
1171         retval = (*dev->dev_ops->dev_close)(dev);
1172         rte_cryptodev_trace_close(dev_id, retval);
1173
1174         if (retval < 0)
1175                 return retval;
1176
1177         return 0;
1178 }
1179
1180 int
1181 rte_cryptodev_get_qp_status(uint8_t dev_id, uint16_t queue_pair_id)
1182 {
1183         struct rte_cryptodev *dev;
1184
1185         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1186                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1187                 return -EINVAL;
1188         }
1189
1190         dev = &rte_crypto_devices[dev_id];
1191         if (queue_pair_id >= dev->data->nb_queue_pairs) {
1192                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
1193                 return -EINVAL;
1194         }
1195         void **qps = dev->data->queue_pairs;
1196
1197         if (qps[queue_pair_id]) {
1198                 CDEV_LOG_DEBUG("qp %d on dev %d is initialised",
1199                         queue_pair_id, dev_id);
1200                 return 1;
1201         }
1202
1203         CDEV_LOG_DEBUG("qp %d on dev %d is not initialised",
1204                 queue_pair_id, dev_id);
1205
1206         return 0;
1207 }
1208
1209 int
1210 rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
1211                 const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
1212
1213 {
1214         struct rte_cryptodev *dev;
1215
1216         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1217                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1218                 return -EINVAL;
1219         }
1220
1221         dev = &rte_crypto_devices[dev_id];
1222         if (queue_pair_id >= dev->data->nb_queue_pairs) {
1223                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
1224                 return -EINVAL;
1225         }
1226
1227         if (!qp_conf) {
1228                 CDEV_LOG_ERR("qp_conf cannot be NULL\n");
1229                 return -EINVAL;
1230         }
1231
1232         if ((qp_conf->mp_session && !qp_conf->mp_session_private) ||
1233                         (!qp_conf->mp_session && qp_conf->mp_session_private)) {
1234                 CDEV_LOG_ERR("Invalid mempools\n");
1235                 return -EINVAL;
1236         }
1237
1238         if (qp_conf->mp_session) {
1239                 struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1240                 uint32_t obj_size = qp_conf->mp_session->elt_size;
1241                 uint32_t obj_priv_size = qp_conf->mp_session_private->elt_size;
1242                 struct rte_cryptodev_sym_session s = {0};
1243
1244                 pool_priv = rte_mempool_get_priv(qp_conf->mp_session);
1245                 if (!pool_priv || qp_conf->mp_session->private_data_size <
1246                                 sizeof(*pool_priv)) {
1247                         CDEV_LOG_ERR("Invalid mempool\n");
1248                         return -EINVAL;
1249                 }
1250
1251                 s.nb_drivers = pool_priv->nb_drivers;
1252                 s.user_data_sz = pool_priv->user_data_sz;
1253
1254                 if ((rte_cryptodev_sym_get_existing_header_session_size(&s) >
1255                         obj_size) || (s.nb_drivers <= dev->driver_id) ||
1256                         rte_cryptodev_sym_get_private_session_size(dev_id) >
1257                                 obj_priv_size) {
1258                         CDEV_LOG_ERR("Invalid mempool\n");
1259                         return -EINVAL;
1260                 }
1261         }
1262
1263         if (dev->data->dev_started) {
1264                 CDEV_LOG_ERR(
1265                     "device %d must be stopped to allow configuration", dev_id);
1266                 return -EBUSY;
1267         }
1268
1269         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
1270
1271         rte_cryptodev_trace_queue_pair_setup(dev_id, queue_pair_id, qp_conf);
1272         return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id, qp_conf,
1273                         socket_id);
1274 }
1275
1276 struct rte_cryptodev_cb *
1277 rte_cryptodev_add_enq_callback(uint8_t dev_id,
1278                                uint16_t qp_id,
1279                                rte_cryptodev_callback_fn cb_fn,
1280                                void *cb_arg)
1281 {
1282         struct rte_cryptodev *dev;
1283         struct rte_cryptodev_cb_rcu *list;
1284         struct rte_cryptodev_cb *cb, *tail;
1285
1286         if (!cb_fn) {
1287                 CDEV_LOG_ERR("Callback is NULL on dev_id=%d", dev_id);
1288                 rte_errno = EINVAL;
1289                 return NULL;
1290         }
1291
1292         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1293                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1294                 rte_errno = ENODEV;
1295                 return NULL;
1296         }
1297
1298         dev = &rte_crypto_devices[dev_id];
1299         if (qp_id >= dev->data->nb_queue_pairs) {
1300                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1301                 rte_errno = ENODEV;
1302                 return NULL;
1303         }
1304
1305         cb = rte_zmalloc(NULL, sizeof(*cb), 0);
1306         if (cb == NULL) {
1307                 CDEV_LOG_ERR("Failed to allocate memory for callback on "
1308                              "dev=%d, queue_pair_id=%d", dev_id, qp_id);
1309                 rte_errno = ENOMEM;
1310                 return NULL;
1311         }
1312
1313         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1314
1315         cb->fn = cb_fn;
1316         cb->arg = cb_arg;
1317
1318         /* Add the callbacks in fifo order. */
1319         list = &dev->enq_cbs[qp_id];
1320         tail = list->next;
1321
1322         if (tail) {
1323                 while (tail->next)
1324                         tail = tail->next;
1325                 /* Stores to cb->fn and cb->param should complete before
1326                  * cb is visible to data plane.
1327                  */
1328                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
1329         } else {
1330                 /* Stores to cb->fn and cb->param should complete before
1331                  * cb is visible to data plane.
1332                  */
1333                 __atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
1334         }
1335
1336         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1337
1338         return cb;
1339 }
1340
1341 int
1342 rte_cryptodev_remove_enq_callback(uint8_t dev_id,
1343                                   uint16_t qp_id,
1344                                   struct rte_cryptodev_cb *cb)
1345 {
1346         struct rte_cryptodev *dev;
1347         struct rte_cryptodev_cb **prev_cb, *curr_cb;
1348         struct rte_cryptodev_cb_rcu *list;
1349         int ret;
1350
1351         ret = -EINVAL;
1352
1353         if (!cb) {
1354                 CDEV_LOG_ERR("Callback is NULL");
1355                 return -EINVAL;
1356         }
1357
1358         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1359                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1360                 return -ENODEV;
1361         }
1362
1363         dev = &rte_crypto_devices[dev_id];
1364         if (qp_id >= dev->data->nb_queue_pairs) {
1365                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1366                 return -ENODEV;
1367         }
1368
1369         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1370         if (dev->enq_cbs == NULL) {
1371                 CDEV_LOG_ERR("Callback not initialized");
1372                 goto cb_err;
1373         }
1374
1375         list = &dev->enq_cbs[qp_id];
1376         if (list == NULL) {
1377                 CDEV_LOG_ERR("Callback list is NULL");
1378                 goto cb_err;
1379         }
1380
1381         if (list->qsbr == NULL) {
1382                 CDEV_LOG_ERR("Rcu qsbr is NULL");
1383                 goto cb_err;
1384         }
1385
1386         prev_cb = &list->next;
1387         for (; *prev_cb != NULL; prev_cb = &curr_cb->next) {
1388                 curr_cb = *prev_cb;
1389                 if (curr_cb == cb) {
1390                         /* Remove the user cb from the callback list. */
1391                         __atomic_store_n(prev_cb, curr_cb->next,
1392                                 __ATOMIC_RELAXED);
1393                         ret = 0;
1394                         break;
1395                 }
1396         }
1397
1398         if (!ret) {
1399                 /* Call sync with invalid thread id as this is part of
1400                  * control plane API
1401                  */
1402                 rte_rcu_qsbr_synchronize(list->qsbr, RTE_QSBR_THRID_INVALID);
1403                 rte_free(cb);
1404         }
1405
1406 cb_err:
1407         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1408         return ret;
1409 }
1410
1411 struct rte_cryptodev_cb *
1412 rte_cryptodev_add_deq_callback(uint8_t dev_id,
1413                                uint16_t qp_id,
1414                                rte_cryptodev_callback_fn cb_fn,
1415                                void *cb_arg)
1416 {
1417         struct rte_cryptodev *dev;
1418         struct rte_cryptodev_cb_rcu *list;
1419         struct rte_cryptodev_cb *cb, *tail;
1420
1421         if (!cb_fn) {
1422                 CDEV_LOG_ERR("Callback is NULL on dev_id=%d", dev_id);
1423                 rte_errno = EINVAL;
1424                 return NULL;
1425         }
1426
1427         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1428                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1429                 rte_errno = ENODEV;
1430                 return NULL;
1431         }
1432
1433         dev = &rte_crypto_devices[dev_id];
1434         if (qp_id >= dev->data->nb_queue_pairs) {
1435                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1436                 rte_errno = ENODEV;
1437                 return NULL;
1438         }
1439
1440         cb = rte_zmalloc(NULL, sizeof(*cb), 0);
1441         if (cb == NULL) {
1442                 CDEV_LOG_ERR("Failed to allocate memory for callback on "
1443                              "dev=%d, queue_pair_id=%d", dev_id, qp_id);
1444                 rte_errno = ENOMEM;
1445                 return NULL;
1446         }
1447
1448         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1449
1450         cb->fn = cb_fn;
1451         cb->arg = cb_arg;
1452
1453         /* Add the callbacks in fifo order. */
1454         list = &dev->deq_cbs[qp_id];
1455         tail = list->next;
1456
1457         if (tail) {
1458                 while (tail->next)
1459                         tail = tail->next;
1460                 /* Stores to cb->fn and cb->param should complete before
1461                  * cb is visible to data plane.
1462                  */
1463                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
1464         } else {
1465                 /* Stores to cb->fn and cb->param should complete before
1466                  * cb is visible to data plane.
1467                  */
1468                 __atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
1469         }
1470
1471         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1472
1473         return cb;
1474 }
1475
1476 int
1477 rte_cryptodev_remove_deq_callback(uint8_t dev_id,
1478                                   uint16_t qp_id,
1479                                   struct rte_cryptodev_cb *cb)
1480 {
1481         struct rte_cryptodev *dev;
1482         struct rte_cryptodev_cb **prev_cb, *curr_cb;
1483         struct rte_cryptodev_cb_rcu *list;
1484         int ret;
1485
1486         ret = -EINVAL;
1487
1488         if (!cb) {
1489                 CDEV_LOG_ERR("Callback is NULL");
1490                 return -EINVAL;
1491         }
1492
1493         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1494                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1495                 return -ENODEV;
1496         }
1497
1498         dev = &rte_crypto_devices[dev_id];
1499         if (qp_id >= dev->data->nb_queue_pairs) {
1500                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", qp_id);
1501                 return -ENODEV;
1502         }
1503
1504         rte_spinlock_lock(&rte_cryptodev_callback_lock);
1505         if (dev->enq_cbs == NULL) {
1506                 CDEV_LOG_ERR("Callback not initialized");
1507                 goto cb_err;
1508         }
1509
1510         list = &dev->deq_cbs[qp_id];
1511         if (list == NULL) {
1512                 CDEV_LOG_ERR("Callback list is NULL");
1513                 goto cb_err;
1514         }
1515
1516         if (list->qsbr == NULL) {
1517                 CDEV_LOG_ERR("Rcu qsbr is NULL");
1518                 goto cb_err;
1519         }
1520
1521         prev_cb = &list->next;
1522         for (; *prev_cb != NULL; prev_cb = &curr_cb->next) {
1523                 curr_cb = *prev_cb;
1524                 if (curr_cb == cb) {
1525                         /* Remove the user cb from the callback list. */
1526                         __atomic_store_n(prev_cb, curr_cb->next,
1527                                 __ATOMIC_RELAXED);
1528                         ret = 0;
1529                         break;
1530                 }
1531         }
1532
1533         if (!ret) {
1534                 /* Call sync with invalid thread id as this is part of
1535                  * control plane API
1536                  */
1537                 rte_rcu_qsbr_synchronize(list->qsbr, RTE_QSBR_THRID_INVALID);
1538                 rte_free(cb);
1539         }
1540
1541 cb_err:
1542         rte_spinlock_unlock(&rte_cryptodev_callback_lock);
1543         return ret;
1544 }
1545
1546 int
1547 rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats)
1548 {
1549         struct rte_cryptodev *dev;
1550
1551         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1552                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1553                 return -ENODEV;
1554         }
1555
1556         if (stats == NULL) {
1557                 CDEV_LOG_ERR("Invalid stats ptr");
1558                 return -EINVAL;
1559         }
1560
1561         dev = &rte_crypto_devices[dev_id];
1562         memset(stats, 0, sizeof(*stats));
1563
1564         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1565         (*dev->dev_ops->stats_get)(dev, stats);
1566         return 0;
1567 }
1568
1569 void
1570 rte_cryptodev_stats_reset(uint8_t dev_id)
1571 {
1572         struct rte_cryptodev *dev;
1573
1574         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1575                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1576                 return;
1577         }
1578
1579         dev = &rte_crypto_devices[dev_id];
1580
1581         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
1582         (*dev->dev_ops->stats_reset)(dev);
1583 }
1584
1585 void
1586 rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
1587 {
1588         struct rte_cryptodev *dev;
1589
1590         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1591                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1592                 return;
1593         }
1594
1595         dev = &rte_crypto_devices[dev_id];
1596
1597         memset(dev_info, 0, sizeof(struct rte_cryptodev_info));
1598
1599         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
1600         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
1601
1602         dev_info->driver_name = dev->device->driver->name;
1603         dev_info->device = dev->device;
1604 }
1605
1606 int
1607 rte_cryptodev_callback_register(uint8_t dev_id,
1608                         enum rte_cryptodev_event_type event,
1609                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
1610 {
1611         struct rte_cryptodev *dev;
1612         struct rte_cryptodev_callback *user_cb;
1613
1614         if (!cb_fn)
1615                 return -EINVAL;
1616
1617         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1618                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1619                 return -EINVAL;
1620         }
1621
1622         dev = &rte_crypto_devices[dev_id];
1623         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1624
1625         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
1626                 if (user_cb->cb_fn == cb_fn &&
1627                         user_cb->cb_arg == cb_arg &&
1628                         user_cb->event == event) {
1629                         break;
1630                 }
1631         }
1632
1633         /* create a new callback. */
1634         if (user_cb == NULL) {
1635                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
1636                                 sizeof(struct rte_cryptodev_callback), 0);
1637                 if (user_cb != NULL) {
1638                         user_cb->cb_fn = cb_fn;
1639                         user_cb->cb_arg = cb_arg;
1640                         user_cb->event = event;
1641                         TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
1642                 }
1643         }
1644
1645         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1646         return (user_cb == NULL) ? -ENOMEM : 0;
1647 }
1648
1649 int
1650 rte_cryptodev_callback_unregister(uint8_t dev_id,
1651                         enum rte_cryptodev_event_type event,
1652                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
1653 {
1654         int ret;
1655         struct rte_cryptodev *dev;
1656         struct rte_cryptodev_callback *cb, *next;
1657
1658         if (!cb_fn)
1659                 return -EINVAL;
1660
1661         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1662                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1663                 return -EINVAL;
1664         }
1665
1666         dev = &rte_crypto_devices[dev_id];
1667         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1668
1669         ret = 0;
1670         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
1671
1672                 next = TAILQ_NEXT(cb, next);
1673
1674                 if (cb->cb_fn != cb_fn || cb->event != event ||
1675                                 (cb->cb_arg != (void *)-1 &&
1676                                 cb->cb_arg != cb_arg))
1677                         continue;
1678
1679                 /*
1680                  * if this callback is not executing right now,
1681                  * then remove it.
1682                  */
1683                 if (cb->active == 0) {
1684                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
1685                         rte_free(cb);
1686                 } else {
1687                         ret = -EAGAIN;
1688                 }
1689         }
1690
1691         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1692         return ret;
1693 }
1694
1695 void
1696 rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev,
1697         enum rte_cryptodev_event_type event)
1698 {
1699         struct rte_cryptodev_callback *cb_lst;
1700         struct rte_cryptodev_callback dev_cb;
1701
1702         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1703         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
1704                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1705                         continue;
1706                 dev_cb = *cb_lst;
1707                 cb_lst->active = 1;
1708                 rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1709                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1710                                                 dev_cb.cb_arg);
1711                 rte_spinlock_lock(&rte_cryptodev_cb_lock);
1712                 cb_lst->active = 0;
1713         }
1714         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1715 }
1716
1717 int
1718 rte_cryptodev_sym_session_init(uint8_t dev_id,
1719                 struct rte_cryptodev_sym_session *sess,
1720                 struct rte_crypto_sym_xform *xforms,
1721                 struct rte_mempool *mp)
1722 {
1723         struct rte_cryptodev *dev;
1724         uint32_t sess_priv_sz = rte_cryptodev_sym_get_private_session_size(
1725                         dev_id);
1726         uint8_t index;
1727         int ret;
1728
1729         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1730                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1731                 return -EINVAL;
1732         }
1733
1734         dev = rte_cryptodev_pmd_get_dev(dev_id);
1735
1736         if (sess == NULL || xforms == NULL || dev == NULL || mp == NULL)
1737                 return -EINVAL;
1738
1739         if (mp->elt_size < sess_priv_sz)
1740                 return -EINVAL;
1741
1742         index = dev->driver_id;
1743         if (index >= sess->nb_drivers)
1744                 return -EINVAL;
1745
1746         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_configure, -ENOTSUP);
1747
1748         if (sess->sess_data[index].refcnt == 0) {
1749                 ret = dev->dev_ops->sym_session_configure(dev, xforms,
1750                                                         sess, mp);
1751                 if (ret < 0) {
1752                         CDEV_LOG_ERR(
1753                                 "dev_id %d failed to configure session details",
1754                                 dev_id);
1755                         return ret;
1756                 }
1757         }
1758
1759         rte_cryptodev_trace_sym_session_init(dev_id, sess, xforms, mp);
1760         sess->sess_data[index].refcnt++;
1761         return 0;
1762 }
1763
1764 struct rte_mempool *
1765 rte_cryptodev_sym_session_pool_create(const char *name, uint32_t nb_elts,
1766         uint32_t elt_size, uint32_t cache_size, uint16_t user_data_size,
1767         int socket_id)
1768 {
1769         struct rte_mempool *mp;
1770         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1771         uint32_t obj_sz;
1772
1773         obj_sz = rte_cryptodev_sym_get_header_session_size() + user_data_size;
1774         if (obj_sz > elt_size)
1775                 CDEV_LOG_INFO("elt_size %u is expanded to %u\n", elt_size,
1776                                 obj_sz);
1777         else
1778                 obj_sz = elt_size;
1779
1780         mp = rte_mempool_create(name, nb_elts, obj_sz, cache_size,
1781                         (uint32_t)(sizeof(*pool_priv)),
1782                         NULL, NULL, NULL, NULL,
1783                         socket_id, 0);
1784         if (mp == NULL) {
1785                 CDEV_LOG_ERR("%s(name=%s) failed, rte_errno=%d\n",
1786                         __func__, name, rte_errno);
1787                 return NULL;
1788         }
1789
1790         pool_priv = rte_mempool_get_priv(mp);
1791         if (!pool_priv) {
1792                 CDEV_LOG_ERR("%s(name=%s) failed to get private data\n",
1793                         __func__, name);
1794                 rte_mempool_free(mp);
1795                 return NULL;
1796         }
1797
1798         pool_priv->nb_drivers = nb_drivers;
1799         pool_priv->user_data_sz = user_data_size;
1800
1801         rte_cryptodev_trace_sym_session_pool_create(name, nb_elts,
1802                 elt_size, cache_size, user_data_size, mp);
1803         return mp;
1804 }
1805
1806 struct rte_mempool *
1807 rte_cryptodev_asym_session_pool_create(const char *name, uint32_t nb_elts,
1808         uint32_t cache_size, uint16_t user_data_size, int socket_id)
1809 {
1810         struct rte_mempool *mp;
1811         struct rte_cryptodev_asym_session_pool_private_data *pool_priv;
1812         uint32_t obj_sz, obj_sz_aligned;
1813         uint8_t dev_id, priv_sz, max_priv_sz = 0;
1814
1815         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++)
1816                 if (rte_cryptodev_is_valid_dev(dev_id)) {
1817                         priv_sz = rte_cryptodev_asym_get_private_session_size(dev_id);
1818                         if (priv_sz > max_priv_sz)
1819                                 max_priv_sz = priv_sz;
1820                 }
1821         if (max_priv_sz == 0) {
1822                 CDEV_LOG_INFO("Could not set max private session size\n");
1823                 return NULL;
1824         }
1825
1826         obj_sz = rte_cryptodev_asym_get_header_session_size() + max_priv_sz +
1827                         user_data_size;
1828         obj_sz_aligned =  RTE_ALIGN_CEIL(obj_sz, RTE_CACHE_LINE_SIZE);
1829
1830         mp = rte_mempool_create(name, nb_elts, obj_sz_aligned, cache_size,
1831                         (uint32_t)(sizeof(*pool_priv)),
1832                         NULL, NULL, NULL, NULL,
1833                         socket_id, 0);
1834         if (mp == NULL) {
1835                 CDEV_LOG_ERR("%s(name=%s) failed, rte_errno=%d\n",
1836                         __func__, name, rte_errno);
1837                 return NULL;
1838         }
1839
1840         pool_priv = rte_mempool_get_priv(mp);
1841         if (!pool_priv) {
1842                 CDEV_LOG_ERR("%s(name=%s) failed to get private data\n",
1843                         __func__, name);
1844                 rte_mempool_free(mp);
1845                 return NULL;
1846         }
1847         pool_priv->max_priv_session_sz = max_priv_sz;
1848         pool_priv->user_data_sz = user_data_size;
1849
1850         rte_cryptodev_trace_asym_session_pool_create(name, nb_elts,
1851                 user_data_size, cache_size, mp);
1852         return mp;
1853 }
1854
1855 static unsigned int
1856 rte_cryptodev_sym_session_data_size(struct rte_cryptodev_sym_session *sess)
1857 {
1858         return (sizeof(sess->sess_data[0]) * sess->nb_drivers) +
1859                         sess->user_data_sz;
1860 }
1861
1862 static uint8_t
1863 rte_cryptodev_sym_is_valid_session_pool(struct rte_mempool *mp)
1864 {
1865         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1866
1867         if (!mp)
1868                 return 0;
1869
1870         pool_priv = rte_mempool_get_priv(mp);
1871
1872         if (!pool_priv || mp->private_data_size < sizeof(*pool_priv) ||
1873                         pool_priv->nb_drivers != nb_drivers ||
1874                         mp->elt_size <
1875                                 rte_cryptodev_sym_get_header_session_size()
1876                                 + pool_priv->user_data_sz)
1877                 return 0;
1878
1879         return 1;
1880 }
1881
1882 struct rte_cryptodev_sym_session *
1883 rte_cryptodev_sym_session_create(struct rte_mempool *mp)
1884 {
1885         struct rte_cryptodev_sym_session *sess;
1886         struct rte_cryptodev_sym_session_pool_private_data *pool_priv;
1887
1888         if (!rte_cryptodev_sym_is_valid_session_pool(mp)) {
1889                 CDEV_LOG_ERR("Invalid mempool\n");
1890                 return NULL;
1891         }
1892
1893         pool_priv = rte_mempool_get_priv(mp);
1894
1895         /* Allocate a session structure from the session pool */
1896         if (rte_mempool_get(mp, (void **)&sess)) {
1897                 CDEV_LOG_ERR("couldn't get object from session mempool");
1898                 return NULL;
1899         }
1900
1901         sess->nb_drivers = pool_priv->nb_drivers;
1902         sess->user_data_sz = pool_priv->user_data_sz;
1903         sess->opaque_data = 0;
1904
1905         /* Clear device session pointer.
1906          * Include the flag indicating presence of user data
1907          */
1908         memset(sess->sess_data, 0,
1909                         rte_cryptodev_sym_session_data_size(sess));
1910
1911         rte_cryptodev_trace_sym_session_create(mp, sess);
1912         return sess;
1913 }
1914
1915 int
1916 rte_cryptodev_asym_session_create(uint8_t dev_id,
1917                 struct rte_crypto_asym_xform *xforms, struct rte_mempool *mp,
1918                 void **session)
1919 {
1920         struct rte_cryptodev_asym_session *sess;
1921         uint32_t session_priv_data_sz;
1922         struct rte_cryptodev_asym_session_pool_private_data *pool_priv;
1923         unsigned int session_header_size =
1924                         rte_cryptodev_asym_get_header_session_size();
1925         struct rte_cryptodev *dev;
1926         int ret;
1927
1928         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1929                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1930                 return -EINVAL;
1931         }
1932
1933         dev = rte_cryptodev_pmd_get_dev(dev_id);
1934
1935         if (dev == NULL)
1936                 return -EINVAL;
1937
1938         if (!mp) {
1939                 CDEV_LOG_ERR("invalid mempool\n");
1940                 return -EINVAL;
1941         }
1942
1943         session_priv_data_sz = rte_cryptodev_asym_get_private_session_size(
1944                         dev_id);
1945         pool_priv = rte_mempool_get_priv(mp);
1946
1947         if (pool_priv->max_priv_session_sz < session_priv_data_sz) {
1948                 CDEV_LOG_DEBUG(
1949                         "The private session data size used when creating the mempool is smaller than this device's private session data.");
1950                 return -EINVAL;
1951         }
1952
1953         /* Verify if provided mempool can hold elements big enough. */
1954         if (mp->elt_size < session_header_size + session_priv_data_sz) {
1955                 CDEV_LOG_ERR(
1956                         "mempool elements too small to hold session objects");
1957                 return -EINVAL;
1958         }
1959
1960         /* Allocate a session structure from the session pool */
1961         if (rte_mempool_get(mp, session)) {
1962                 CDEV_LOG_ERR("couldn't get object from session mempool");
1963                 return -ENOMEM;
1964         }
1965
1966         sess = *session;
1967         sess->driver_id = dev->driver_id;
1968         sess->user_data_sz = pool_priv->user_data_sz;
1969         sess->max_priv_data_sz = pool_priv->max_priv_session_sz;
1970
1971         /* Clear device session pointer.*/
1972         memset(sess->sess_private_data, 0, session_priv_data_sz + sess->user_data_sz);
1973
1974         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_configure, -ENOTSUP);
1975
1976         if (sess->sess_private_data[0] == 0) {
1977                 ret = dev->dev_ops->asym_session_configure(dev, xforms, sess);
1978                 if (ret < 0) {
1979                         CDEV_LOG_ERR(
1980                                 "dev_id %d failed to configure session details",
1981                                 dev_id);
1982                         return ret;
1983                 }
1984         }
1985
1986         rte_cryptodev_trace_asym_session_create(dev_id, xforms, mp, sess);
1987         return 0;
1988 }
1989
1990 int
1991 rte_cryptodev_sym_session_clear(uint8_t dev_id,
1992                 struct rte_cryptodev_sym_session *sess)
1993 {
1994         struct rte_cryptodev *dev;
1995         uint8_t driver_id;
1996
1997         if (!rte_cryptodev_is_valid_dev(dev_id)) {
1998                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
1999                 return -EINVAL;
2000         }
2001
2002         dev = rte_cryptodev_pmd_get_dev(dev_id);
2003
2004         if (dev == NULL || sess == NULL)
2005                 return -EINVAL;
2006
2007         driver_id = dev->driver_id;
2008         if (sess->sess_data[driver_id].refcnt == 0)
2009                 return 0;
2010         if (--sess->sess_data[driver_id].refcnt != 0)
2011                 return -EBUSY;
2012
2013         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->sym_session_clear, -ENOTSUP);
2014
2015         dev->dev_ops->sym_session_clear(dev, sess);
2016
2017         rte_cryptodev_trace_sym_session_clear(dev_id, sess);
2018         return 0;
2019 }
2020
2021 int
2022 rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess)
2023 {
2024         uint8_t i;
2025         struct rte_mempool *sess_mp;
2026
2027         if (sess == NULL)
2028                 return -EINVAL;
2029
2030         /* Check that all device private data has been freed */
2031         for (i = 0; i < sess->nb_drivers; i++) {
2032                 if (sess->sess_data[i].refcnt != 0)
2033                         return -EBUSY;
2034         }
2035
2036         /* Return session to mempool */
2037         sess_mp = rte_mempool_from_obj(sess);
2038         rte_mempool_put(sess_mp, sess);
2039
2040         rte_cryptodev_trace_sym_session_free(sess);
2041         return 0;
2042 }
2043
2044 int
2045 rte_cryptodev_asym_session_free(uint8_t dev_id, void *sess)
2046 {
2047         struct rte_mempool *sess_mp;
2048         struct rte_cryptodev *dev;
2049
2050         if (!rte_cryptodev_is_valid_dev(dev_id)) {
2051                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
2052                 return -EINVAL;
2053         }
2054
2055         dev = rte_cryptodev_pmd_get_dev(dev_id);
2056
2057         if (dev == NULL || sess == NULL)
2058                 return -EINVAL;
2059
2060         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_clear, -ENOTSUP);
2061
2062         dev->dev_ops->asym_session_clear(dev, sess);
2063
2064         /* Return session to mempool */
2065         sess_mp = rte_mempool_from_obj(sess);
2066         rte_mempool_put(sess_mp, sess);
2067
2068         rte_cryptodev_trace_asym_session_free(dev_id, sess);
2069         return 0;
2070 }
2071
2072 unsigned int
2073 rte_cryptodev_sym_get_header_session_size(void)
2074 {
2075         /*
2076          * Header contains pointers to the private data of all registered
2077          * drivers and all necessary information to ensure safely clear
2078          * or free al session.
2079          */
2080         struct rte_cryptodev_sym_session s = {0};
2081
2082         s.nb_drivers = nb_drivers;
2083
2084         return (unsigned int)(sizeof(s) +
2085                         rte_cryptodev_sym_session_data_size(&s));
2086 }
2087
2088 unsigned int
2089 rte_cryptodev_sym_get_existing_header_session_size(
2090                 struct rte_cryptodev_sym_session *sess)
2091 {
2092         if (!sess)
2093                 return 0;
2094         else
2095                 return (unsigned int)(sizeof(*sess) +
2096                                 rte_cryptodev_sym_session_data_size(sess));
2097 }
2098
2099 unsigned int
2100 rte_cryptodev_asym_get_header_session_size(void)
2101 {
2102         return sizeof(struct rte_cryptodev_asym_session);
2103 }
2104
2105 unsigned int
2106 rte_cryptodev_sym_get_private_session_size(uint8_t dev_id)
2107 {
2108         struct rte_cryptodev *dev;
2109         unsigned int priv_sess_size;
2110
2111         if (!rte_cryptodev_is_valid_dev(dev_id))
2112                 return 0;
2113
2114         dev = rte_cryptodev_pmd_get_dev(dev_id);
2115
2116         if (*dev->dev_ops->sym_session_get_size == NULL)
2117                 return 0;
2118
2119         priv_sess_size = (*dev->dev_ops->sym_session_get_size)(dev);
2120
2121         return priv_sess_size;
2122 }
2123
2124 unsigned int
2125 rte_cryptodev_asym_get_private_session_size(uint8_t dev_id)
2126 {
2127         struct rte_cryptodev *dev;
2128         unsigned int priv_sess_size;
2129
2130         if (!rte_cryptodev_is_valid_dev(dev_id))
2131                 return 0;
2132
2133         dev = rte_cryptodev_pmd_get_dev(dev_id);
2134
2135         if (*dev->dev_ops->asym_session_get_size == NULL)
2136                 return 0;
2137
2138         priv_sess_size = (*dev->dev_ops->asym_session_get_size)(dev);
2139
2140         return priv_sess_size;
2141 }
2142
2143 int
2144 rte_cryptodev_sym_session_set_user_data(
2145                                         struct rte_cryptodev_sym_session *sess,
2146                                         void *data,
2147                                         uint16_t size)
2148 {
2149         if (sess == NULL)
2150                 return -EINVAL;
2151
2152         if (sess->user_data_sz < size)
2153                 return -ENOMEM;
2154
2155         rte_memcpy(sess->sess_data + sess->nb_drivers, data, size);
2156         return 0;
2157 }
2158
2159 void *
2160 rte_cryptodev_sym_session_get_user_data(
2161                                         struct rte_cryptodev_sym_session *sess)
2162 {
2163         if (sess == NULL || sess->user_data_sz == 0)
2164                 return NULL;
2165
2166         return (void *)(sess->sess_data + sess->nb_drivers);
2167 }
2168
2169 int
2170 rte_cryptodev_asym_session_set_user_data(void *session, void *data, uint16_t size)
2171 {
2172         struct rte_cryptodev_asym_session *sess = session;
2173         if (sess == NULL)
2174                 return -EINVAL;
2175
2176         if (sess->user_data_sz < size)
2177                 return -ENOMEM;
2178
2179         rte_memcpy(sess->sess_private_data +
2180                         sess->max_priv_data_sz,
2181                         data, size);
2182         return 0;
2183 }
2184
2185 void *
2186 rte_cryptodev_asym_session_get_user_data(void *session)
2187 {
2188         struct rte_cryptodev_asym_session *sess = session;
2189         if (sess == NULL || sess->user_data_sz == 0)
2190                 return NULL;
2191
2192         return (void *)(sess->sess_private_data +
2193                         sess->max_priv_data_sz);
2194 }
2195
2196 static inline void
2197 sym_crypto_fill_status(struct rte_crypto_sym_vec *vec, int32_t errnum)
2198 {
2199         uint32_t i;
2200         for (i = 0; i < vec->num; i++)
2201                 vec->status[i] = errnum;
2202 }
2203
2204 uint32_t
2205 rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
2206         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs,
2207         struct rte_crypto_sym_vec *vec)
2208 {
2209         struct rte_cryptodev *dev;
2210
2211         if (!rte_cryptodev_is_valid_dev(dev_id)) {
2212                 sym_crypto_fill_status(vec, EINVAL);
2213                 return 0;
2214         }
2215
2216         dev = rte_cryptodev_pmd_get_dev(dev_id);
2217
2218         if (*dev->dev_ops->sym_cpu_process == NULL ||
2219                 !(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO)) {
2220                 sym_crypto_fill_status(vec, ENOTSUP);
2221                 return 0;
2222         }
2223
2224         return dev->dev_ops->sym_cpu_process(dev, sess, ofs, vec);
2225 }
2226
2227 int
2228 rte_cryptodev_get_raw_dp_ctx_size(uint8_t dev_id)
2229 {
2230         struct rte_cryptodev *dev;
2231         int32_t size = sizeof(struct rte_crypto_raw_dp_ctx);
2232         int32_t priv_size;
2233
2234         if (!rte_cryptodev_is_valid_dev(dev_id))
2235                 return -EINVAL;
2236
2237         dev = rte_cryptodev_pmd_get_dev(dev_id);
2238
2239         if (*dev->dev_ops->sym_get_raw_dp_ctx_size == NULL ||
2240                 !(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)) {
2241                 return -ENOTSUP;
2242         }
2243
2244         priv_size = (*dev->dev_ops->sym_get_raw_dp_ctx_size)(dev);
2245         if (priv_size < 0)
2246                 return -ENOTSUP;
2247
2248         return RTE_ALIGN_CEIL((size + priv_size), 8);
2249 }
2250
2251 int
2252 rte_cryptodev_configure_raw_dp_ctx(uint8_t dev_id, uint16_t qp_id,
2253         struct rte_crypto_raw_dp_ctx *ctx,
2254         enum rte_crypto_op_sess_type sess_type,
2255         union rte_cryptodev_session_ctx session_ctx,
2256         uint8_t is_update)
2257 {
2258         struct rte_cryptodev *dev;
2259
2260         if (!rte_cryptodev_get_qp_status(dev_id, qp_id))
2261                 return -EINVAL;
2262
2263         dev = rte_cryptodev_pmd_get_dev(dev_id);
2264         if (!(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)
2265                         || dev->dev_ops->sym_configure_raw_dp_ctx == NULL)
2266                 return -ENOTSUP;
2267
2268         return (*dev->dev_ops->sym_configure_raw_dp_ctx)(dev, qp_id, ctx,
2269                         sess_type, session_ctx, is_update);
2270 }
2271
2272 uint32_t
2273 rte_cryptodev_raw_enqueue_burst(struct rte_crypto_raw_dp_ctx *ctx,
2274         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
2275         void **user_data, int *enqueue_status)
2276 {
2277         return (*ctx->enqueue_burst)(ctx->qp_data, ctx->drv_ctx_data, vec,
2278                         ofs, user_data, enqueue_status);
2279 }
2280
2281 int
2282 rte_cryptodev_raw_enqueue_done(struct rte_crypto_raw_dp_ctx *ctx,
2283                 uint32_t n)
2284 {
2285         return (*ctx->enqueue_done)(ctx->qp_data, ctx->drv_ctx_data, n);
2286 }
2287
2288 uint32_t
2289 rte_cryptodev_raw_dequeue_burst(struct rte_crypto_raw_dp_ctx *ctx,
2290         rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
2291         uint32_t max_nb_to_dequeue,
2292         rte_cryptodev_raw_post_dequeue_t post_dequeue,
2293         void **out_user_data, uint8_t is_user_data_array,
2294         uint32_t *n_success_jobs, int *status)
2295 {
2296         return (*ctx->dequeue_burst)(ctx->qp_data, ctx->drv_ctx_data,
2297                 get_dequeue_count, max_nb_to_dequeue, post_dequeue,
2298                 out_user_data, is_user_data_array, n_success_jobs, status);
2299 }
2300
2301 int
2302 rte_cryptodev_raw_dequeue_done(struct rte_crypto_raw_dp_ctx *ctx,
2303                 uint32_t n)
2304 {
2305         return (*ctx->dequeue_done)(ctx->qp_data, ctx->drv_ctx_data, n);
2306 }
2307
2308 /** Initialise rte_crypto_op mempool element */
2309 static void
2310 rte_crypto_op_init(struct rte_mempool *mempool,
2311                 void *opaque_arg,
2312                 void *_op_data,
2313                 __rte_unused unsigned i)
2314 {
2315         struct rte_crypto_op *op = _op_data;
2316         enum rte_crypto_op_type type = *(enum rte_crypto_op_type *)opaque_arg;
2317
2318         memset(_op_data, 0, mempool->elt_size);
2319
2320         __rte_crypto_op_reset(op, type);
2321
2322         op->phys_addr = rte_mem_virt2iova(_op_data);
2323         op->mempool = mempool;
2324 }
2325
2326
2327 struct rte_mempool *
2328 rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type,
2329                 unsigned nb_elts, unsigned cache_size, uint16_t priv_size,
2330                 int socket_id)
2331 {
2332         struct rte_crypto_op_pool_private *priv;
2333
2334         unsigned elt_size = sizeof(struct rte_crypto_op) +
2335                         priv_size;
2336
2337         if (type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
2338                 elt_size += sizeof(struct rte_crypto_sym_op);
2339         } else if (type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
2340                 elt_size += sizeof(struct rte_crypto_asym_op);
2341         } else if (type == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
2342                 elt_size += RTE_MAX(sizeof(struct rte_crypto_sym_op),
2343                                     sizeof(struct rte_crypto_asym_op));
2344         } else {
2345                 CDEV_LOG_ERR("Invalid op_type\n");
2346                 return NULL;
2347         }
2348
2349         /* lookup mempool in case already allocated */
2350         struct rte_mempool *mp = rte_mempool_lookup(name);
2351
2352         if (mp != NULL) {
2353                 priv = (struct rte_crypto_op_pool_private *)
2354                                 rte_mempool_get_priv(mp);
2355
2356                 if (mp->elt_size != elt_size ||
2357                                 mp->cache_size < cache_size ||
2358                                 mp->size < nb_elts ||
2359                                 priv->priv_size <  priv_size) {
2360                         mp = NULL;
2361                         CDEV_LOG_ERR("Mempool %s already exists but with "
2362                                         "incompatible parameters", name);
2363                         return NULL;
2364                 }
2365                 return mp;
2366         }
2367
2368         mp = rte_mempool_create(
2369                         name,
2370                         nb_elts,
2371                         elt_size,
2372                         cache_size,
2373                         sizeof(struct rte_crypto_op_pool_private),
2374                         NULL,
2375                         NULL,
2376                         rte_crypto_op_init,
2377                         &type,
2378                         socket_id,
2379                         0);
2380
2381         if (mp == NULL) {
2382                 CDEV_LOG_ERR("Failed to create mempool %s", name);
2383                 return NULL;
2384         }
2385
2386         priv = (struct rte_crypto_op_pool_private *)
2387                         rte_mempool_get_priv(mp);
2388
2389         priv->priv_size = priv_size;
2390         priv->type = type;
2391
2392         return mp;
2393 }
2394
2395 int
2396 rte_cryptodev_pmd_create_dev_name(char *name, const char *dev_name_prefix)
2397 {
2398         struct rte_cryptodev *dev = NULL;
2399         uint32_t i = 0;
2400
2401         if (name == NULL)
2402                 return -EINVAL;
2403
2404         for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) {
2405                 int ret = snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
2406                                 "%s_%u", dev_name_prefix, i);
2407
2408                 if (ret < 0)
2409                         return ret;
2410
2411                 dev = rte_cryptodev_pmd_get_named_dev(name);
2412                 if (!dev)
2413                         return 0;
2414         }
2415
2416         return -1;
2417 }
2418
2419 TAILQ_HEAD(cryptodev_driver_list, cryptodev_driver);
2420
2421 static struct cryptodev_driver_list cryptodev_driver_list =
2422         TAILQ_HEAD_INITIALIZER(cryptodev_driver_list);
2423
2424 int
2425 rte_cryptodev_driver_id_get(const char *name)
2426 {
2427         struct cryptodev_driver *driver;
2428         const char *driver_name;
2429
2430         if (name == NULL) {
2431                 RTE_LOG(DEBUG, CRYPTODEV, "name pointer NULL");
2432                 return -1;
2433         }
2434
2435         TAILQ_FOREACH(driver, &cryptodev_driver_list, next) {
2436                 driver_name = driver->driver->name;
2437                 if (strncmp(driver_name, name, strlen(driver_name) + 1) == 0)
2438                         return driver->id;
2439         }
2440         return -1;
2441 }
2442
2443 const char *
2444 rte_cryptodev_name_get(uint8_t dev_id)
2445 {
2446         struct rte_cryptodev *dev;
2447
2448         if (!rte_cryptodev_is_valid_device_data(dev_id)) {
2449                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
2450                 return NULL;
2451         }
2452
2453         dev = rte_cryptodev_pmd_get_dev(dev_id);
2454         if (dev == NULL)
2455                 return NULL;
2456
2457         return dev->data->name;
2458 }
2459
2460 const char *
2461 rte_cryptodev_driver_name_get(uint8_t driver_id)
2462 {
2463         struct cryptodev_driver *driver;
2464
2465         TAILQ_FOREACH(driver, &cryptodev_driver_list, next)
2466                 if (driver->id == driver_id)
2467                         return driver->driver->name;
2468         return NULL;
2469 }
2470
2471 uint8_t
2472 rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv,
2473                 const struct rte_driver *drv)
2474 {
2475         crypto_drv->driver = drv;
2476         crypto_drv->id = nb_drivers;
2477
2478         TAILQ_INSERT_TAIL(&cryptodev_driver_list, crypto_drv, next);
2479
2480         return nb_drivers++;
2481 }
2482
2483 RTE_INIT(cryptodev_init_fp_ops)
2484 {
2485         uint32_t i;
2486
2487         for (i = 0; i != RTE_DIM(rte_crypto_fp_ops); i++)
2488                 cryptodev_fp_ops_reset(rte_crypto_fp_ops + i);
2489 }
2490
2491 static int
2492 cryptodev_handle_dev_list(const char *cmd __rte_unused,
2493                 const char *params __rte_unused,
2494                 struct rte_tel_data *d)
2495 {
2496         int dev_id;
2497
2498         if (rte_cryptodev_count() < 1)
2499                 return -EINVAL;
2500
2501         rte_tel_data_start_array(d, RTE_TEL_INT_VAL);
2502         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++)
2503                 if (rte_cryptodev_is_valid_dev(dev_id))
2504                         rte_tel_data_add_array_int(d, dev_id);
2505
2506         return 0;
2507 }
2508
2509 static int
2510 cryptodev_handle_dev_info(const char *cmd __rte_unused,
2511                 const char *params, struct rte_tel_data *d)
2512 {
2513         struct rte_cryptodev_info cryptodev_info;
2514         int dev_id;
2515         char *end_param;
2516
2517         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
2518                 return -EINVAL;
2519
2520         dev_id = strtoul(params, &end_param, 0);
2521         if (*end_param != '\0')
2522                 CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
2523         if (!rte_cryptodev_is_valid_dev(dev_id))
2524                 return -EINVAL;
2525
2526         rte_cryptodev_info_get(dev_id, &cryptodev_info);
2527
2528         rte_tel_data_start_dict(d);
2529         rte_tel_data_add_dict_string(d, "device_name",
2530                 cryptodev_info.device->name);
2531         rte_tel_data_add_dict_int(d, "max_nb_queue_pairs",
2532                 cryptodev_info.max_nb_queue_pairs);
2533
2534         return 0;
2535 }
2536
2537 #define ADD_DICT_STAT(s) rte_tel_data_add_dict_u64(d, #s, cryptodev_stats.s)
2538
2539 static int
2540 cryptodev_handle_dev_stats(const char *cmd __rte_unused,
2541                 const char *params,
2542                 struct rte_tel_data *d)
2543 {
2544         struct rte_cryptodev_stats cryptodev_stats;
2545         int dev_id, ret;
2546         char *end_param;
2547
2548         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
2549                 return -EINVAL;
2550
2551         dev_id = strtoul(params, &end_param, 0);
2552         if (*end_param != '\0')
2553                 CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
2554         if (!rte_cryptodev_is_valid_dev(dev_id))
2555                 return -EINVAL;
2556
2557         ret = rte_cryptodev_stats_get(dev_id, &cryptodev_stats);
2558         if (ret < 0)
2559                 return ret;
2560
2561         rte_tel_data_start_dict(d);
2562         ADD_DICT_STAT(enqueued_count);
2563         ADD_DICT_STAT(dequeued_count);
2564         ADD_DICT_STAT(enqueue_err_count);
2565         ADD_DICT_STAT(dequeue_err_count);
2566
2567         return 0;
2568 }
2569
2570 #define CRYPTO_CAPS_SZ                                             \
2571         (RTE_ALIGN_CEIL(sizeof(struct rte_cryptodev_capabilities), \
2572                                         sizeof(uint64_t)) /        \
2573          sizeof(uint64_t))
2574
2575 static int
2576 crypto_caps_array(struct rte_tel_data *d,
2577                   const struct rte_cryptodev_capabilities *capabilities)
2578 {
2579         const struct rte_cryptodev_capabilities *dev_caps;
2580         uint64_t caps_val[CRYPTO_CAPS_SZ];
2581         unsigned int i = 0, j;
2582
2583         rte_tel_data_start_array(d, RTE_TEL_U64_VAL);
2584
2585         while ((dev_caps = &capabilities[i++])->op !=
2586                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
2587                 memset(&caps_val, 0, CRYPTO_CAPS_SZ * sizeof(caps_val[0]));
2588                 rte_memcpy(caps_val, dev_caps, sizeof(capabilities[0]));
2589                 for (j = 0; j < CRYPTO_CAPS_SZ; j++)
2590                         rte_tel_data_add_array_u64(d, caps_val[j]);
2591         }
2592
2593         return i;
2594 }
2595
2596 static int
2597 cryptodev_handle_dev_caps(const char *cmd __rte_unused, const char *params,
2598                           struct rte_tel_data *d)
2599 {
2600         struct rte_cryptodev_info dev_info;
2601         struct rte_tel_data *crypto_caps;
2602         int crypto_caps_n;
2603         char *end_param;
2604         int dev_id;
2605
2606         if (!params || strlen(params) == 0 || !isdigit(*params))
2607                 return -EINVAL;
2608
2609         dev_id = strtoul(params, &end_param, 0);
2610         if (*end_param != '\0')
2611                 CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
2612         if (!rte_cryptodev_is_valid_dev(dev_id))
2613                 return -EINVAL;
2614
2615         rte_tel_data_start_dict(d);
2616         crypto_caps = rte_tel_data_alloc();
2617         if (!crypto_caps)
2618                 return -ENOMEM;
2619
2620         rte_cryptodev_info_get(dev_id, &dev_info);
2621         crypto_caps_n = crypto_caps_array(crypto_caps, dev_info.capabilities);
2622         rte_tel_data_add_dict_container(d, "crypto_caps", crypto_caps, 0);
2623         rte_tel_data_add_dict_int(d, "crypto_caps_n", crypto_caps_n);
2624
2625         return 0;
2626 }
2627
2628 RTE_INIT(cryptodev_init_telemetry)
2629 {
2630         rte_telemetry_register_cmd("/cryptodev/info", cryptodev_handle_dev_info,
2631                         "Returns information for a cryptodev. Parameters: int dev_id");
2632         rte_telemetry_register_cmd("/cryptodev/list",
2633                         cryptodev_handle_dev_list,
2634                         "Returns list of available crypto devices by IDs. No parameters.");
2635         rte_telemetry_register_cmd("/cryptodev/stats",
2636                         cryptodev_handle_dev_stats,
2637                         "Returns the stats for a cryptodev. Parameters: int dev_id");
2638         rte_telemetry_register_cmd("/cryptodev/caps",
2639                         cryptodev_handle_dev_caps,
2640                         "Returns the capabilities for a cryptodev. Parameters: int dev_id");
2641 }