drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_malloc.h>
   9 #include <rte_cryptodev_pmd.h>
  10
  11 #include "rte_aesni_mb_pmd_private.h"
  12
  13
  14 static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
  15         {       /* MD5 HMAC */
  16                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  17                 {.sym = {
  18                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  19                         {.auth = {
  20                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
  21                                 .block_size = 64,
  22                                 .key_size = {
  23                                         .min = 1,
  24                                         .max = 64,
  25                                         .increment = 1
  26                                 },
  27                                 .digest_size = {
  28                                         .min = 1,
  29                                         .max = 16,
  30                                         .increment = 1
  31                                 },
  32                                 .iv_size = { 0 }
  33                         }, }
  34                 }, }
  35         },
  36         {       /* SHA1 HMAC */
  37                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  38                 {.sym = {
  39                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  40                         {.auth = {
  41                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  42                                 .block_size = 64,
  43                                 .key_size = {
  44                                         .min = 1,
  45                                         .max = 65535,
  46                                         .increment = 1
  47                                 },
  48                                 .digest_size = {
  49                                         .min = 1,
  50                                         .max = 20,
  51                                         .increment = 1
  52                                 },
  53                                 .iv_size = { 0 }
  54                         }, }
  55                 }, }
  56         },
  57         {       /* SHA1 */
  58                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  59                 {.sym = {
  60                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  61                         {.auth = {
  62                                 .algo = RTE_CRYPTO_AUTH_SHA1,
  63                                 .block_size = 64,
  64                                 .key_size = {
  65                                         .min = 0,
  66                                         .max = 0,
  67                                         .increment = 0
  68                                 },
  69                                 .digest_size = {
  70                                         .min = 1,
  71                                         .max = 20,
  72                                         .increment = 1
  73                                 },
  74                                 .iv_size = { 0 }
  75                         }, }
  76                 }, }
  77         },
  78         {       /* SHA224 HMAC */
  79                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  80                 {.sym = {
  81                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  82                         {.auth = {
  83                                 .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
  84                                 .block_size = 64,
  85                                 .key_size = {
  86                                         .min = 1,
  87                                         .max = 65535,
  88                                         .increment = 1
  89                                 },
  90                                 .digest_size = {
  91                                         .min = 1,
  92                                         .max = 28,
  93                                         .increment = 1
  94                                 },
  95                                 .iv_size = { 0 }
  96                         }, }
  97                 }, }
  98         },
  99         {       /* SHA224 */
 100                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 101                 {.sym = {
 102                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 103                         {.auth = {
 104                                 .algo = RTE_CRYPTO_AUTH_SHA224,
 105                                 .block_size = 64,
 106                                 .key_size = {
 107                                         .min = 0,
 108                                         .max = 0,
 109                                         .increment = 0
 110                                 },
 111                                 .digest_size = {
 112                                         .min = 1,
 113                                         .max = 28,
 114                                         .increment = 1
 115                                 },
 116                                 .iv_size = { 0 }
 117                         }, }
 118                 }, }
 119         },
 120         {       /* SHA256 HMAC */
 121                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 122                 {.sym = {
 123                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 124                         {.auth = {
 125                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
 126                                 .block_size = 64,
 127                                 .key_size = {
 128                                         .min = 1,
 129                                         .max = 65535,
 130                                         .increment = 1
 131                                 },
 132                                 .digest_size = {
 133                                         .min = 1,
 134                                         .max = 32,
 135                                         .increment = 1
 136                                 },
 137                                 .iv_size = { 0 }
 138                         }, }
 139                 }, }
 140         },
 141         {       /* SHA256 */
 142                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 143                 {.sym = {
 144                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 145                         {.auth = {
 146                                 .algo = RTE_CRYPTO_AUTH_SHA256,
 147                                 .block_size = 64,
 148                                 .key_size = {
 149                                         .min = 0,
 150                                         .max = 0,
 151                                         .increment = 0
 152                                 },
 153                                 .digest_size = {
 154                                         .min = 1,
 155                                         .max = 32,
 156                                         .increment = 1
 157                                 },
 158                                 .iv_size = { 0 }
 159                         }, }
 160                 }, }
 161         },
 162         {       /* SHA384 HMAC */
 163                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 164                 {.sym = {
 165                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 166                         {.auth = {
 167                                 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
 168                                 .block_size = 128,
 169                                 .key_size = {
 170                                         .min = 1,
 171                                         .max = 65535,
 172                                         .increment = 1
 173                                 },
 174                                 .digest_size = {
 175                                         .min = 1,
 176                                         .max = 48,
 177                                         .increment = 1
 178                                 },
 179                                 .iv_size = { 0 }
 180                         }, }
 181                 }, }
 182         },
 183         {       /* SHA384 */
 184                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 185                 {.sym = {
 186                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 187                         {.auth = {
 188                                 .algo = RTE_CRYPTO_AUTH_SHA384,
 189                                 .block_size = 128,
 190                                 .key_size = {
 191                                         .min = 0,
 192                                         .max = 0,
 193                                         .increment = 0
 194                                 },
 195                                 .digest_size = {
 196                                         .min = 1,
 197                                         .max = 48,
 198                                         .increment = 1
 199                                 },
 200                                 .iv_size = { 0 }
 201                         }, }
 202                 }, }
 203         },
 204         {       /* SHA512 HMAC */
 205                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 206                 {.sym = {
 207                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 208                         {.auth = {
 209                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 210                                 .block_size = 128,
 211                                 .key_size = {
 212                                         .min = 1,
 213                                         .max = 65535,
 214                                         .increment = 1
 215                                 },
 216                                 .digest_size = {
 217                                         .min = 1,
 218                                         .max = 64,
 219                                         .increment = 1
 220                                 },
 221                                 .iv_size = { 0 }
 222                         }, }
 223                 }, }
 224         },
 225         {       /* SHA512  */
 226                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 227                 {.sym = {
 228                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 229                         {.auth = {
 230                                 .algo = RTE_CRYPTO_AUTH_SHA512,
 231                                 .block_size = 128,
 232                                 .key_size = {
 233                                         .min = 0,
 234                                         .max = 0,
 235                                         .increment = 0
 236                                 },
 237                                 .digest_size = {
 238                                         .min = 1,
 239                                         .max = 64,
 240                                         .increment = 1
 241                                 },
 242                                 .iv_size = { 0 }
 243                         }, }
 244                 }, }
 245         },
 246         {       /* AES XCBC HMAC */
 247                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 248                 {.sym = {
 249                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 250                         {.auth = {
 251                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 252                                 .block_size = 16,
 253                                 .key_size = {
 254                                         .min = 16,
 255                                         .max = 16,
 256                                         .increment = 0
 257                                 },
 258                                 .digest_size = {
 259                                         .min = 12,
 260                                         .max = 12,
 261                                         .increment = 0
 262                                 },
 263                                 .iv_size = { 0 }
 264                         }, }
 265                 }, }
 266         },
 267         {       /* AES CBC */
 268                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 269                 {.sym = {
 270                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 271                         {.cipher = {
 272                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 273                                 .block_size = 16,
 274                                 .key_size = {
 275                                         .min = 16,
 276                                         .max = 32,
 277                                         .increment = 8
 278                                 },
 279                                 .iv_size = {
 280                                         .min = 16,
 281                                         .max = 16,
 282                                         .increment = 0
 283                                 }
 284                         }, }
 285                 }, }
 286         },
 287         {       /* AES CTR */
 288                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 289                 {.sym = {
 290                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 291                         {.cipher = {
 292                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 293                                 .block_size = 16,
 294                                 .key_size = {
 295                                         .min = 16,
 296                                         .max = 32,
 297                                         .increment = 8
 298                                 },
 299                                 .iv_size = {
 300                                         .min = 12,
 301                                         .max = 16,
 302                                         .increment = 4
 303                                 }
 304                         }, }
 305                 }, }
 306         },
 307         {       /* AES DOCSIS BPI */
 308                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 309                 {.sym = {
 310                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 311                         {.cipher = {
 312                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 313                                 .block_size = 16,
 314                                 .key_size = {
 315                                         .min = 16,
 316                                         .max = 16,
 317                                         .increment = 0
 318                                 },
 319                                 .iv_size = {
 320                                         .min = 16,
 321                                         .max = 16,
 322                                         .increment = 0
 323                                 }
 324                         }, }
 325                 }, }
 326         },
 327         {       /* DES CBC */
 328                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 329                 {.sym = {
 330                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 331                         {.cipher = {
 332                                 .algo = RTE_CRYPTO_CIPHER_DES_CBC,
 333                                 .block_size = 8,
 334                                 .key_size = {
 335                                         .min = 8,
 336                                         .max = 8,
 337                                         .increment = 0
 338                                 },
 339                                 .iv_size = {
 340                                         .min = 8,
 341                                         .max = 8,
 342                                         .increment = 0
 343                                 }
 344                         }, }
 345                 }, }
 346         },
 347         {       /*  3DES CBC */
 348                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 349                 {.sym = {
 350                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 351                         {.cipher = {
 352                                 .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
 353                                 .block_size = 8,
 354                                 .key_size = {
 355                                         .min = 8,
 356                                         .max = 24,
 357                                         .increment = 8
 358                                 },
 359                                 .iv_size = {
 360                                         .min = 8,
 361                                         .max = 8,
 362                                         .increment = 0
 363                                 }
 364                         }, }
 365                 }, }
 366         },
 367         {       /* DES DOCSIS BPI */
 368                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 369                 {.sym = {
 370                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 371                         {.cipher = {
 372                                 .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
 373                                 .block_size = 8,
 374                                 .key_size = {
 375                                         .min = 8,
 376                                         .max = 8,
 377                                         .increment = 0
 378                                 },
 379                                 .iv_size = {
 380                                         .min = 8,
 381                                         .max = 8,
 382                                         .increment = 0
 383                                 }
 384                         }, }
 385                 }, }
 386         },
 387         {       /* AES CCM */
 388                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 389                 {.sym = {
 390                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 391                         {.aead = {
 392                                 .algo = RTE_CRYPTO_AEAD_AES_CCM,
 393                                 .block_size = 16,
 394                                 .key_size = {
 395                                         .min = 16,
 396                                         .max = 16,
 397                                         .increment = 0
 398                                 },
 399                                 .digest_size = {
 400                                         .min = 4,
 401                                         .max = 16,
 402                                         .increment = 2
 403                                 },
 404                                 .aad_size = {
 405                                         .min = 0,
 406                                         .max = 46,
 407                                         .increment = 1
 408                                 },
 409                                 .iv_size = {
 410                                         .min = 7,
 411                                         .max = 13,
 412                                         .increment = 1
 413                                 },
 414                         }, }
 415                 }, }
 416         },
 417         {       /* AES CMAC */
 418                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 419                 {.sym = {
 420                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 421                         {.auth = {
 422                                 .algo = RTE_CRYPTO_AUTH_AES_CMAC,
 423                                 .block_size = 16,
 424                                 .key_size = {
 425                                         .min = 16,
 426                                         .max = 16,
 427                                         .increment = 0
 428                                 },
 429                                 .digest_size = {
 430                                         .min = 1,
 431                                         .max = 16,
 432                                         .increment = 1
 433                                 },
 434                                 .iv_size = { 0 }
 435                         }, }
 436                 }, }
 437         },
 438         {       /* AES GCM */
 439                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 440                 {.sym = {
 441                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 442                         {.aead = {
 443                                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
 444                                 .block_size = 16,
 445                                 .key_size = {
 446                                         .min = 16,
 447                                         .max = 32,
 448                                         .increment = 8
 449                                 },
 450                                 .digest_size = {
 451                                         .min = 8,
 452                                         .max = 16,
 453                                         .increment = 4
 454                                 },
 455                                 .aad_size = {
 456                                         .min = 0,
 457                                         .max = 65535,
 458                                         .increment = 1
 459                                 },
 460                                 .iv_size = {
 461                                         .min = 12,
 462                                         .max = 12,
 463                                         .increment = 0
 464                                 }
 465                         }, }
 466                 }, }
 467         },
 468         {       /* AES GMAC (AUTH) */
 469                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 470                 {.sym = {
 471                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 472                         {.auth = {
 473                                 .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 474                                 .block_size = 16,
 475                                 .key_size = {
 476                                         .min = 16,
 477                                         .max = 32,
 478                                         .increment = 8
 479                                 },
 480                                 .digest_size = {
 481                                         .min = 8,
 482                                         .max = 16,
 483                                         .increment = 4
 484                                 },
 485                                 .iv_size = {
 486                                         .min = 12,
 487                                         .max = 12,
 488                                         .increment = 0
 489                                 }
 490                         }, }
 491                 }, }
 492         },
 493         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 494 };
 495
 496
 497 /** Configure device */
 498 static int
 499 aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
 500                 __rte_unused struct rte_cryptodev_config *config)
 501 {
 502         return 0;
 503 }
 504
 505 /** Start device */
 506 static int
 507 aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
 508 {
 509         return 0;
 510 }
 511
 512 /** Stop device */
 513 static void
 514 aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
 515 {
 516 }
 517
 518 /** Close device */
 519 static int
 520 aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
 521 {
 522         return 0;
 523 }
 524
 525
 526 /** Get device statistics */
 527 static void
 528 aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
 529                 struct rte_cryptodev_stats *stats)
 530 {
 531         int qp_id;
 532
 533         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 534                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 535
 536                 stats->enqueued_count += qp->stats.enqueued_count;
 537                 stats->dequeued_count += qp->stats.dequeued_count;
 538
 539                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 540                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 541         }
 542 }
 543
 544 /** Reset device statistics */
 545 static void
 546 aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
 547 {
 548         int qp_id;
 549
 550         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 551                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 552
 553                 memset(&qp->stats, 0, sizeof(qp->stats));
 554         }
 555 }
 556
 557
 558 /** Get device info */
 559 static void
 560 aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
 561                 struct rte_cryptodev_info *dev_info)
 562 {
 563         struct aesni_mb_private *internals = dev->data->dev_private;
 564
 565         if (dev_info != NULL) {
 566                 dev_info->driver_id = dev->driver_id;
 567                 dev_info->feature_flags = dev->feature_flags;
 568                 dev_info->capabilities = aesni_mb_pmd_capabilities;
 569                 dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
 570                 /* No limit of number of sessions */
 571                 dev_info->sym.max_nb_sessions = 0;
 572         }
 573 }
 574
 575 /** Release queue pair */
 576 static int
 577 aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 578 {
 579         struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 580         struct rte_ring *r = NULL;
 581
 582         if (qp != NULL) {
 583                 r = rte_ring_lookup(qp->name);
 584                 if (r)
 585                         rte_ring_free(r);
 586                 if (qp->mb_mgr)
 587                         free_mb_mgr(qp->mb_mgr);
 588                 rte_free(qp);
 589                 dev->data->queue_pairs[qp_id] = NULL;
 590         }
 591         return 0;
 592 }
 593
 594 /** set a unique name for the queue pair based on it's name, dev_id and qp_id */
 595 static int
 596 aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
 597                 struct aesni_mb_qp *qp)
 598 {
 599         unsigned n = snprintf(qp->name, sizeof(qp->name),
 600                         "aesni_mb_pmd_%u_qp_%u",
 601                         dev->data->dev_id, qp->id);
 602
 603         if (n >= sizeof(qp->name))
 604                 return -1;
 605
 606         return 0;
 607 }
 608
 609 /** Create a ring to place processed operations on */
 610 static struct rte_ring *
 611 aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
 612                 unsigned int ring_size, int socket_id)
 613 {
 614         struct rte_ring *r;
 615         char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 616
 617         unsigned int n = snprintf(ring_name, sizeof(ring_name), "%s", qp->name);
 618
 619         if (n >= sizeof(ring_name))
 620                 return NULL;
 621
 622         r = rte_ring_lookup(ring_name);
 623         if (r) {
 624                 if (rte_ring_get_size(r) >= ring_size) {
 625                         AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
 626                         ring_name);
 627                         return r;
 628                 }
 629
 630                 AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
 631                         ring_name);
 632                 return NULL;
 633         }
 634
 635         return rte_ring_create(ring_name, ring_size, socket_id,
 636                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 637 }
 638
 639 /** Setup a queue pair */
 640 static int
 641 aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 642                 const struct rte_cryptodev_qp_conf *qp_conf,
 643                 int socket_id)
 644 {
 645         struct aesni_mb_qp *qp = NULL;
 646         struct aesni_mb_private *internals = dev->data->dev_private;
 647         int ret = -1;
 648
 649         /* Free memory prior to re-allocation if needed. */
 650         if (dev->data->queue_pairs[qp_id] != NULL)
 651                 aesni_mb_pmd_qp_release(dev, qp_id);
 652
 653         /* Allocate the queue pair data structure. */
 654         qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
 655                                         RTE_CACHE_LINE_SIZE, socket_id);
 656         if (qp == NULL)
 657                 return -ENOMEM;
 658
 659         qp->id = qp_id;
 660         dev->data->queue_pairs[qp_id] = qp;
 661
 662         if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
 663                 goto qp_setup_cleanup;
 664
 665
 666         qp->mb_mgr = alloc_mb_mgr(0);
 667         if (qp->mb_mgr == NULL) {
 668                 ret = -ENOMEM;
 669                 goto qp_setup_cleanup;
 670         }
 671
 672         switch (internals->vector_mode) {
 673         case RTE_AESNI_MB_SSE:
 674                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
 675                 init_mb_mgr_sse(qp->mb_mgr);
 676                 break;
 677         case RTE_AESNI_MB_AVX:
 678                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
 679                 init_mb_mgr_avx(qp->mb_mgr);
 680                 break;
 681         case RTE_AESNI_MB_AVX2:
 682                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 683                 init_mb_mgr_avx2(qp->mb_mgr);
 684                 break;
 685         case RTE_AESNI_MB_AVX512:
 686                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
 687                 init_mb_mgr_avx512(qp->mb_mgr);
 688                 break;
 689         default:
 690                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
 691                                 internals->vector_mode);
 692                 goto qp_setup_cleanup;
 693         }
 694
 695         qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
 696                         qp_conf->nb_descriptors, socket_id);
 697         if (qp->ingress_queue == NULL) {
 698                 ret = -1;
 699                 goto qp_setup_cleanup;
 700         }
 701
 702         qp->sess_mp = qp_conf->mp_session;
 703         qp->sess_mp_priv = qp_conf->mp_session_private;
 704
 705         memset(&qp->stats, 0, sizeof(qp->stats));
 706
 707         char mp_name[RTE_MEMPOOL_NAMESIZE];
 708
 709         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 710                                 "digest_mp_%u_%u", dev->data->dev_id, qp_id);
 711         return 0;
 712
 713 qp_setup_cleanup:
 714         if (qp) {
 715                 if (qp->mb_mgr)
 716                         free_mb_mgr(qp->mb_mgr);
 717                 rte_free(qp);
 718         }
 719
 720         return ret;
 721 }
 722
 723 /** Return the number of allocated queue pairs */
 724 static uint32_t
 725 aesni_mb_pmd_qp_count(struct rte_cryptodev *dev)
 726 {
 727         return dev->data->nb_queue_pairs;
 728 }
 729
 730 /** Returns the size of the aesni multi-buffer session structure */
 731 static unsigned
 732 aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 733 {
 734         return sizeof(struct aesni_mb_session);
 735 }
 736
 737 /** Configure a aesni multi-buffer session from a crypto xform chain */
 738 static int
 739 aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
 740                 struct rte_crypto_sym_xform *xform,
 741                 struct rte_cryptodev_sym_session *sess,
 742                 struct rte_mempool *mempool)
 743 {
 744         void *sess_private_data;
 745         struct aesni_mb_private *internals = dev->data->dev_private;
 746         int ret;
 747
 748         if (unlikely(sess == NULL)) {
 749                 AESNI_MB_LOG(ERR, "invalid session struct");
 750                 return -EINVAL;
 751         }
 752
 753         if (rte_mempool_get(mempool, &sess_private_data)) {
 754                 AESNI_MB_LOG(ERR,
 755                                 "Couldn't get object from session mempool");
 756                 return -ENOMEM;
 757         }
 758
 759         ret = aesni_mb_set_session_parameters(internals->mb_mgr,
 760                         sess_private_data, xform);
 761         if (ret != 0) {
 762                 AESNI_MB_LOG(ERR, "failed configure session parameters");
 763
 764                 /* Return session to mempool */
 765                 rte_mempool_put(mempool, sess_private_data);
 766                 return ret;
 767         }
 768
 769         set_sym_session_private_data(sess, dev->driver_id,
 770                         sess_private_data);
 771
 772         return 0;
 773 }
 774
 775 /** Clear the memory of session so it doesn't leave key material behind */
 776 static void
 777 aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
 778                 struct rte_cryptodev_sym_session *sess)
 779 {
 780         uint8_t index = dev->driver_id;
 781         void *sess_priv = get_sym_session_private_data(sess, index);
 782
 783         /* Zero out the whole structure */
 784         if (sess_priv) {
 785                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
 786                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 787                 set_sym_session_private_data(sess, index, NULL);
 788                 rte_mempool_put(sess_mp, sess_priv);
 789         }
 790 }
 791
 792 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
 793                 .dev_configure          = aesni_mb_pmd_config,
 794                 .dev_start              = aesni_mb_pmd_start,
 795                 .dev_stop               = aesni_mb_pmd_stop,
 796                 .dev_close              = aesni_mb_pmd_close,
 797
 798                 .stats_get              = aesni_mb_pmd_stats_get,
 799                 .stats_reset            = aesni_mb_pmd_stats_reset,
 800
 801                 .dev_infos_get          = aesni_mb_pmd_info_get,
 802
 803                 .queue_pair_setup       = aesni_mb_pmd_qp_setup,
 804                 .queue_pair_release     = aesni_mb_pmd_qp_release,
 805                 .queue_pair_count       = aesni_mb_pmd_qp_count,
 806
 807                 .sym_session_get_size   = aesni_mb_pmd_sym_session_get_size,
 808                 .sym_session_configure  = aesni_mb_pmd_sym_session_configure,
 809                 .sym_session_clear      = aesni_mb_pmd_sym_session_clear
 810 };
 811
 812 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;