drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_string_fns.h>
   8 #include <rte_common.h>
   9 #include <rte_malloc.h>
  10 #include <rte_ether.h>
  11 #include <rte_cryptodev_pmd.h>
  12
  13 #include "aesni_mb_pmd_private.h"
  14
  15
  16 static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
  17         {       /* MD5 HMAC */
  18                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  19                 {.sym = {
  20                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  21                         {.auth = {
  22                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
  23                                 .block_size = 64,
  24                                 .key_size = {
  25                                         .min = 1,
  26                                         .max = 64,
  27                                         .increment = 1
  28                                 },
  29                                 .digest_size = {
  30                                         .min = 1,
  31                                         .max = 16,
  32                                         .increment = 1
  33                                 },
  34                                 .iv_size = { 0 }
  35                         }, }
  36                 }, }
  37         },
  38         {       /* SHA1 HMAC */
  39                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  40                 {.sym = {
  41                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  42                         {.auth = {
  43                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  44                                 .block_size = 64,
  45                                 .key_size = {
  46                                         .min = 1,
  47                                         .max = 65535,
  48                                         .increment = 1
  49                                 },
  50                                 .digest_size = {
  51                                         .min = 1,
  52                                         .max = 20,
  53                                         .increment = 1
  54                                 },
  55                                 .iv_size = { 0 }
  56                         }, }
  57                 }, }
  58         },
  59         {       /* SHA1 */
  60                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  61                 {.sym = {
  62                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  63                         {.auth = {
  64                                 .algo = RTE_CRYPTO_AUTH_SHA1,
  65                                 .block_size = 64,
  66                                 .key_size = {
  67                                         .min = 0,
  68                                         .max = 0,
  69                                         .increment = 0
  70                                 },
  71                                 .digest_size = {
  72                                         .min = 1,
  73                                         .max = 20,
  74                                         .increment = 1
  75                                 },
  76                                 .iv_size = { 0 }
  77                         }, }
  78                 }, }
  79         },
  80         {       /* SHA224 HMAC */
  81                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  82                 {.sym = {
  83                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  84                         {.auth = {
  85                                 .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
  86                                 .block_size = 64,
  87                                 .key_size = {
  88                                         .min = 1,
  89                                         .max = 65535,
  90                                         .increment = 1
  91                                 },
  92                                 .digest_size = {
  93                                         .min = 1,
  94                                         .max = 28,
  95                                         .increment = 1
  96                                 },
  97                                 .iv_size = { 0 }
  98                         }, }
  99                 }, }
 100         },
 101         {       /* SHA224 */
 102                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 103                 {.sym = {
 104                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 105                         {.auth = {
 106                                 .algo = RTE_CRYPTO_AUTH_SHA224,
 107                                 .block_size = 64,
 108                                 .key_size = {
 109                                         .min = 0,
 110                                         .max = 0,
 111                                         .increment = 0
 112                                 },
 113                                 .digest_size = {
 114                                         .min = 1,
 115                                         .max = 28,
 116                                         .increment = 1
 117                                 },
 118                                 .iv_size = { 0 }
 119                         }, }
 120                 }, }
 121         },
 122         {       /* SHA256 HMAC */
 123                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 124                 {.sym = {
 125                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 126                         {.auth = {
 127                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
 128                                 .block_size = 64,
 129                                 .key_size = {
 130                                         .min = 1,
 131                                         .max = 65535,
 132                                         .increment = 1
 133                                 },
 134                                 .digest_size = {
 135                                         .min = 1,
 136                                         .max = 32,
 137                                         .increment = 1
 138                                 },
 139                                 .iv_size = { 0 }
 140                         }, }
 141                 }, }
 142         },
 143         {       /* SHA256 */
 144                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 145                 {.sym = {
 146                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 147                         {.auth = {
 148                                 .algo = RTE_CRYPTO_AUTH_SHA256,
 149                                 .block_size = 64,
 150                                 .key_size = {
 151                                         .min = 0,
 152                                         .max = 0,
 153                                         .increment = 0
 154                                 },
 155                                 .digest_size = {
 156                                         .min = 1,
 157                                         .max = 32,
 158                                         .increment = 1
 159                                 },
 160                                 .iv_size = { 0 }
 161                         }, }
 162                 }, }
 163         },
 164         {       /* SHA384 HMAC */
 165                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 166                 {.sym = {
 167                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 168                         {.auth = {
 169                                 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
 170                                 .block_size = 128,
 171                                 .key_size = {
 172                                         .min = 1,
 173                                         .max = 65535,
 174                                         .increment = 1
 175                                 },
 176                                 .digest_size = {
 177                                         .min = 1,
 178                                         .max = 48,
 179                                         .increment = 1
 180                                 },
 181                                 .iv_size = { 0 }
 182                         }, }
 183                 }, }
 184         },
 185         {       /* SHA384 */
 186                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 187                 {.sym = {
 188                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 189                         {.auth = {
 190                                 .algo = RTE_CRYPTO_AUTH_SHA384,
 191                                 .block_size = 128,
 192                                 .key_size = {
 193                                         .min = 0,
 194                                         .max = 0,
 195                                         .increment = 0
 196                                 },
 197                                 .digest_size = {
 198                                         .min = 1,
 199                                         .max = 48,
 200                                         .increment = 1
 201                                 },
 202                                 .iv_size = { 0 }
 203                         }, }
 204                 }, }
 205         },
 206         {       /* SHA512 HMAC */
 207                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 208                 {.sym = {
 209                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 210                         {.auth = {
 211                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 212                                 .block_size = 128,
 213                                 .key_size = {
 214                                         .min = 1,
 215                                         .max = 65535,
 216                                         .increment = 1
 217                                 },
 218                                 .digest_size = {
 219                                         .min = 1,
 220                                         .max = 64,
 221                                         .increment = 1
 222                                 },
 223                                 .iv_size = { 0 }
 224                         }, }
 225                 }, }
 226         },
 227         {       /* SHA512  */
 228                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 229                 {.sym = {
 230                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 231                         {.auth = {
 232                                 .algo = RTE_CRYPTO_AUTH_SHA512,
 233                                 .block_size = 128,
 234                                 .key_size = {
 235                                         .min = 0,
 236                                         .max = 0,
 237                                         .increment = 0
 238                                 },
 239                                 .digest_size = {
 240                                         .min = 1,
 241                                         .max = 64,
 242                                         .increment = 1
 243                                 },
 244                                 .iv_size = { 0 }
 245                         }, }
 246                 }, }
 247         },
 248         {       /* AES XCBC HMAC */
 249                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 250                 {.sym = {
 251                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 252                         {.auth = {
 253                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 254                                 .block_size = 16,
 255                                 .key_size = {
 256                                         .min = 16,
 257                                         .max = 16,
 258                                         .increment = 0
 259                                 },
 260                                 .digest_size = {
 261                                         .min = 12,
 262                                         .max = 12,
 263                                         .increment = 0
 264                                 },
 265                                 .iv_size = { 0 }
 266                         }, }
 267                 }, }
 268         },
 269         {       /* AES CBC */
 270                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 271                 {.sym = {
 272                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 273                         {.cipher = {
 274                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 275                                 .block_size = 16,
 276                                 .key_size = {
 277                                         .min = 16,
 278                                         .max = 32,
 279                                         .increment = 8
 280                                 },
 281                                 .iv_size = {
 282                                         .min = 16,
 283                                         .max = 16,
 284                                         .increment = 0
 285                                 }
 286                         }, }
 287                 }, }
 288         },
 289         {       /* AES CTR */
 290                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 291                 {.sym = {
 292                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 293                         {.cipher = {
 294                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 295                                 .block_size = 16,
 296                                 .key_size = {
 297                                         .min = 16,
 298                                         .max = 32,
 299                                         .increment = 8
 300                                 },
 301                                 .iv_size = {
 302                                         .min = 12,
 303                                         .max = 16,
 304                                         .increment = 4
 305                                 }
 306                         }, }
 307                 }, }
 308         },
 309         {       /* AES DOCSIS BPI */
 310                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 311                 {.sym = {
 312                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 313                         {.cipher = {
 314                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 315                                 .block_size = 16,
 316                                 .key_size = {
 317                                         .min = 16,
 318 #if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
 319                                         .max = 32,
 320                                         .increment = 16
 321 #else
 322                                         .max = 16,
 323                                         .increment = 0
 324 #endif
 325                                 },
 326                                 .iv_size = {
 327                                         .min = 16,
 328                                         .max = 16,
 329                                         .increment = 0
 330                                 }
 331                         }, }
 332                 }, }
 333         },
 334         {       /* DES CBC */
 335                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 336                 {.sym = {
 337                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 338                         {.cipher = {
 339                                 .algo = RTE_CRYPTO_CIPHER_DES_CBC,
 340                                 .block_size = 8,
 341                                 .key_size = {
 342                                         .min = 8,
 343                                         .max = 8,
 344                                         .increment = 0
 345                                 },
 346                                 .iv_size = {
 347                                         .min = 8,
 348                                         .max = 8,
 349                                         .increment = 0
 350                                 }
 351                         }, }
 352                 }, }
 353         },
 354         {       /*  3DES CBC */
 355                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 356                 {.sym = {
 357                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 358                         {.cipher = {
 359                                 .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
 360                                 .block_size = 8,
 361                                 .key_size = {
 362                                         .min = 8,
 363                                         .max = 24,
 364                                         .increment = 8
 365                                 },
 366                                 .iv_size = {
 367                                         .min = 8,
 368                                         .max = 8,
 369                                         .increment = 0
 370                                 }
 371                         }, }
 372                 }, }
 373         },
 374         {       /* DES DOCSIS BPI */
 375                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 376                 {.sym = {
 377                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 378                         {.cipher = {
 379                                 .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
 380                                 .block_size = 8,
 381                                 .key_size = {
 382                                         .min = 8,
 383                                         .max = 8,
 384                                         .increment = 0
 385                                 },
 386                                 .iv_size = {
 387                                         .min = 8,
 388                                         .max = 8,
 389                                         .increment = 0
 390                                 }
 391                         }, }
 392                 }, }
 393         },
 394         {       /* AES CCM */
 395                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 396                 {.sym = {
 397                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 398                         {.aead = {
 399                                 .algo = RTE_CRYPTO_AEAD_AES_CCM,
 400                                 .block_size = 16,
 401                                 .key_size = {
 402                                         .min = 16,
 403                                         .max = 16,
 404                                         .increment = 0
 405                                 },
 406                                 .digest_size = {
 407                                         .min = 4,
 408                                         .max = 16,
 409                                         .increment = 2
 410                                 },
 411                                 .aad_size = {
 412                                         .min = 0,
 413                                         .max = 46,
 414                                         .increment = 1
 415                                 },
 416                                 .iv_size = {
 417                                         .min = 7,
 418                                         .max = 13,
 419                                         .increment = 1
 420                                 },
 421                         }, }
 422                 }, }
 423         },
 424         {       /* AES CMAC */
 425                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 426                 {.sym = {
 427                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 428                         {.auth = {
 429                                 .algo = RTE_CRYPTO_AUTH_AES_CMAC,
 430                                 .block_size = 16,
 431                                 .key_size = {
 432                                         .min = 16,
 433                                         .max = 16,
 434                                         .increment = 0
 435                                 },
 436                                 .digest_size = {
 437                                         .min = 1,
 438                                         .max = 16,
 439                                         .increment = 1
 440                                 },
 441                                 .iv_size = { 0 }
 442                         }, }
 443                 }, }
 444         },
 445         {       /* AES GCM */
 446                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 447                 {.sym = {
 448                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 449                         {.aead = {
 450                                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
 451                                 .block_size = 16,
 452                                 .key_size = {
 453                                         .min = 16,
 454                                         .max = 32,
 455                                         .increment = 8
 456                                 },
 457                                 .digest_size = {
 458                                         .min = 8,
 459                                         .max = 16,
 460                                         .increment = 4
 461                                 },
 462                                 .aad_size = {
 463                                         .min = 0,
 464                                         .max = 65535,
 465                                         .increment = 1
 466                                 },
 467                                 .iv_size = {
 468                                         .min = 12,
 469                                         .max = 12,
 470                                         .increment = 0
 471                                 }
 472                         }, }
 473                 }, }
 474         },
 475         {       /* AES GMAC (AUTH) */
 476                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 477                 {.sym = {
 478                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 479                         {.auth = {
 480                                 .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 481                                 .block_size = 16,
 482                                 .key_size = {
 483                                         .min = 16,
 484                                         .max = 32,
 485                                         .increment = 8
 486                                 },
 487                                 .digest_size = {
 488                                         .min = 8,
 489                                         .max = 16,
 490                                         .increment = 4
 491                                 },
 492                                 .iv_size = {
 493                                         .min = 12,
 494                                         .max = 12,
 495                                         .increment = 0
 496                                 }
 497                         }, }
 498                 }, }
 499         },
 500         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 501 };
 502
 503 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 504 static const struct rte_cryptodev_capabilities
 505                                         aesni_mb_pmd_security_crypto_cap[] = {
 506         {       /* AES DOCSIS BPI */
 507                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 508                 {.sym = {
 509                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 510                         {.cipher = {
 511                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 512                                 .block_size = 16,
 513                                 .key_size = {
 514                                         .min = 16,
 515                                         .max = 32,
 516                                         .increment = 16
 517                                 },
 518                                 .iv_size = {
 519                                         .min = 16,
 520                                         .max = 16,
 521                                         .increment = 0
 522                                 }
 523                         }, }
 524                 }, }
 525         },
 526
 527         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 528 };
 529
 530 static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
 531         {       /* DOCSIS Uplink */
 532                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 533                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 534                 .docsis = {
 535                         .direction = RTE_SECURITY_DOCSIS_UPLINK
 536                 },
 537                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 538         },
 539         {       /* DOCSIS Downlink */
 540                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 541                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 542                 .docsis = {
 543                         .direction = RTE_SECURITY_DOCSIS_DOWNLINK
 544                 },
 545                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 546         },
 547         {
 548                 .action = RTE_SECURITY_ACTION_TYPE_NONE
 549         }
 550 };
 551 #endif
 552
 553 /** Configure device */
 554 static int
 555 aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
 556                 __rte_unused struct rte_cryptodev_config *config)
 557 {
 558         return 0;
 559 }
 560
 561 /** Start device */
 562 static int
 563 aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
 564 {
 565         return 0;
 566 }
 567
 568 /** Stop device */
 569 static void
 570 aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
 571 {
 572 }
 573
 574 /** Close device */
 575 static int
 576 aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
 577 {
 578         return 0;
 579 }
 580
 581
 582 /** Get device statistics */
 583 static void
 584 aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
 585                 struct rte_cryptodev_stats *stats)
 586 {
 587         int qp_id;
 588
 589         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 590                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 591
 592                 stats->enqueued_count += qp->stats.enqueued_count;
 593                 stats->dequeued_count += qp->stats.dequeued_count;
 594
 595                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 596                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 597         }
 598 }
 599
 600 /** Reset device statistics */
 601 static void
 602 aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
 603 {
 604         int qp_id;
 605
 606         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 607                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 608
 609                 memset(&qp->stats, 0, sizeof(qp->stats));
 610         }
 611 }
 612
 613
 614 /** Get device info */
 615 static void
 616 aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
 617                 struct rte_cryptodev_info *dev_info)
 618 {
 619         struct aesni_mb_private *internals = dev->data->dev_private;
 620
 621         if (dev_info != NULL) {
 622                 dev_info->driver_id = dev->driver_id;
 623                 dev_info->feature_flags = dev->feature_flags;
 624                 dev_info->capabilities = aesni_mb_pmd_capabilities;
 625                 dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
 626                 /* No limit of number of sessions */
 627                 dev_info->sym.max_nb_sessions = 0;
 628         }
 629 }
 630
 631 /** Release queue pair */
 632 static int
 633 aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 634 {
 635         struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 636         struct rte_ring *r = NULL;
 637
 638         if (qp != NULL) {
 639                 r = rte_ring_lookup(qp->name);
 640                 if (r)
 641                         rte_ring_free(r);
 642                 if (qp->mb_mgr)
 643                         free_mb_mgr(qp->mb_mgr);
 644                 rte_free(qp);
 645                 dev->data->queue_pairs[qp_id] = NULL;
 646         }
 647         return 0;
 648 }
 649
 650 /** set a unique name for the queue pair based on it's name, dev_id and qp_id */
 651 static int
 652 aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
 653                 struct aesni_mb_qp *qp)
 654 {
 655         unsigned n = snprintf(qp->name, sizeof(qp->name),
 656                         "aesni_mb_pmd_%u_qp_%u",
 657                         dev->data->dev_id, qp->id);
 658
 659         if (n >= sizeof(qp->name))
 660                 return -1;
 661
 662         return 0;
 663 }
 664
 665 /** Create a ring to place processed operations on */
 666 static struct rte_ring *
 667 aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
 668                 unsigned int ring_size, int socket_id)
 669 {
 670         struct rte_ring *r;
 671         char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 672
 673         unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
 674
 675         if (n >= sizeof(ring_name))
 676                 return NULL;
 677
 678         r = rte_ring_lookup(ring_name);
 679         if (r) {
 680                 if (rte_ring_get_size(r) >= ring_size) {
 681                         AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
 682                         ring_name);
 683                         return r;
 684                 }
 685
 686                 AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
 687                         ring_name);
 688                 return NULL;
 689         }
 690
 691         return rte_ring_create(ring_name, ring_size, socket_id,
 692                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 693 }
 694
 695 /** Setup a queue pair */
 696 static int
 697 aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 698                 const struct rte_cryptodev_qp_conf *qp_conf,
 699                 int socket_id)
 700 {
 701         struct aesni_mb_qp *qp = NULL;
 702         struct aesni_mb_private *internals = dev->data->dev_private;
 703         int ret = -1;
 704
 705         /* Free memory prior to re-allocation if needed. */
 706         if (dev->data->queue_pairs[qp_id] != NULL)
 707                 aesni_mb_pmd_qp_release(dev, qp_id);
 708
 709         /* Allocate the queue pair data structure. */
 710         qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
 711                                         RTE_CACHE_LINE_SIZE, socket_id);
 712         if (qp == NULL)
 713                 return -ENOMEM;
 714
 715         qp->id = qp_id;
 716         dev->data->queue_pairs[qp_id] = qp;
 717
 718         if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
 719                 goto qp_setup_cleanup;
 720
 721
 722         qp->mb_mgr = alloc_mb_mgr(0);
 723         if (qp->mb_mgr == NULL) {
 724                 ret = -ENOMEM;
 725                 goto qp_setup_cleanup;
 726         }
 727
 728         switch (internals->vector_mode) {
 729         case RTE_AESNI_MB_SSE:
 730                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
 731                 init_mb_mgr_sse(qp->mb_mgr);
 732                 break;
 733         case RTE_AESNI_MB_AVX:
 734                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
 735                 init_mb_mgr_avx(qp->mb_mgr);
 736                 break;
 737         case RTE_AESNI_MB_AVX2:
 738                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 739                 init_mb_mgr_avx2(qp->mb_mgr);
 740                 break;
 741         case RTE_AESNI_MB_AVX512:
 742                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
 743                 init_mb_mgr_avx512(qp->mb_mgr);
 744                 break;
 745         default:
 746                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
 747                                 internals->vector_mode);
 748                 goto qp_setup_cleanup;
 749         }
 750
 751         qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
 752                         qp_conf->nb_descriptors, socket_id);
 753         if (qp->ingress_queue == NULL) {
 754                 ret = -1;
 755                 goto qp_setup_cleanup;
 756         }
 757
 758         qp->sess_mp = qp_conf->mp_session;
 759         qp->sess_mp_priv = qp_conf->mp_session_private;
 760
 761         memset(&qp->stats, 0, sizeof(qp->stats));
 762
 763         char mp_name[RTE_MEMPOOL_NAMESIZE];
 764
 765         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 766                                 "digest_mp_%u_%u", dev->data->dev_id, qp_id);
 767         return 0;
 768
 769 qp_setup_cleanup:
 770         if (qp) {
 771                 if (qp->mb_mgr)
 772                         free_mb_mgr(qp->mb_mgr);
 773                 rte_free(qp);
 774         }
 775
 776         return ret;
 777 }
 778
 779 /** Returns the size of the aesni multi-buffer session structure */
 780 static unsigned
 781 aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 782 {
 783         return sizeof(struct aesni_mb_session);
 784 }
 785
 786 /** Configure a aesni multi-buffer session from a crypto xform chain */
 787 static int
 788 aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
 789                 struct rte_crypto_sym_xform *xform,
 790                 struct rte_cryptodev_sym_session *sess,
 791                 struct rte_mempool *mempool)
 792 {
 793         void *sess_private_data;
 794         struct aesni_mb_private *internals = dev->data->dev_private;
 795         int ret;
 796
 797         if (unlikely(sess == NULL)) {
 798                 AESNI_MB_LOG(ERR, "invalid session struct");
 799                 return -EINVAL;
 800         }
 801
 802         if (rte_mempool_get(mempool, &sess_private_data)) {
 803                 AESNI_MB_LOG(ERR,
 804                                 "Couldn't get object from session mempool");
 805                 return -ENOMEM;
 806         }
 807
 808         ret = aesni_mb_set_session_parameters(internals->mb_mgr,
 809                         sess_private_data, xform);
 810         if (ret != 0) {
 811                 AESNI_MB_LOG(ERR, "failed configure session parameters");
 812
 813                 /* Return session to mempool */
 814                 rte_mempool_put(mempool, sess_private_data);
 815                 return ret;
 816         }
 817
 818         set_sym_session_private_data(sess, dev->driver_id,
 819                         sess_private_data);
 820
 821         return 0;
 822 }
 823
 824 /** Clear the memory of session so it doesn't leave key material behind */
 825 static void
 826 aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
 827                 struct rte_cryptodev_sym_session *sess)
 828 {
 829         uint8_t index = dev->driver_id;
 830         void *sess_priv = get_sym_session_private_data(sess, index);
 831
 832         /* Zero out the whole structure */
 833         if (sess_priv) {
 834                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
 835                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 836                 set_sym_session_private_data(sess, index, NULL);
 837                 rte_mempool_put(sess_mp, sess_priv);
 838         }
 839 }
 840
 841 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
 842                 .dev_configure          = aesni_mb_pmd_config,
 843                 .dev_start              = aesni_mb_pmd_start,
 844                 .dev_stop               = aesni_mb_pmd_stop,
 845                 .dev_close              = aesni_mb_pmd_close,
 846
 847                 .stats_get              = aesni_mb_pmd_stats_get,
 848                 .stats_reset            = aesni_mb_pmd_stats_reset,
 849
 850                 .dev_infos_get          = aesni_mb_pmd_info_get,
 851
 852                 .queue_pair_setup       = aesni_mb_pmd_qp_setup,
 853                 .queue_pair_release     = aesni_mb_pmd_qp_release,
 854
 855                 .sym_cpu_process        = aesni_mb_cpu_crypto_process_bulk,
 856
 857                 .sym_session_get_size   = aesni_mb_pmd_sym_session_get_size,
 858                 .sym_session_configure  = aesni_mb_pmd_sym_session_configure,
 859                 .sym_session_clear      = aesni_mb_pmd_sym_session_clear
 860 };
 861
 862 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
 863
 864 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 865 /**
 866  * Configure a aesni multi-buffer session from a security session
 867  * configuration
 868  */
 869 static int
 870 aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
 871                 struct rte_security_session *sess,
 872                 struct rte_mempool *mempool)
 873 {
 874         void *sess_private_data;
 875         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
 876         int ret;
 877
 878         if (rte_mempool_get(mempool, &sess_private_data)) {
 879                 AESNI_MB_LOG(ERR, "Couldn't get object from session mempool");
 880                 return -ENOMEM;
 881         }
 882
 883         if (conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
 884                 AESNI_MB_LOG(ERR, "Invalid security protocol");
 885                 return -EINVAL;
 886         }
 887
 888         ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
 889                         sess_private_data);
 890
 891         if (ret != 0) {
 892                 AESNI_MB_LOG(ERR, "Failed to configure session parameters");
 893
 894                 /* Return session to mempool */
 895                 rte_mempool_put(mempool, sess_private_data);
 896                 return ret;
 897         }
 898
 899         set_sec_session_private_data(sess, sess_private_data);
 900
 901         return ret;
 902 }
 903
 904 /** Clear the memory of session so it doesn't leave key material behind */
 905 static int
 906 aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
 907                 struct rte_security_session *sess)
 908 {
 909         void *sess_priv = get_sec_session_private_data(sess);
 910
 911         if (sess_priv) {
 912                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 913                 memset(sess, 0, sizeof(struct aesni_mb_session));
 914                 set_sec_session_private_data(sess, NULL);
 915                 rte_mempool_put(sess_mp, sess_priv);
 916         }
 917         return 0;
 918 }
 919
 920 /** Get security capabilities for aesni multi-buffer */
 921 static const struct rte_security_capability *
 922 aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
 923 {
 924         return aesni_mb_pmd_security_cap;
 925 }
 926
 927 static struct rte_security_ops aesni_mb_pmd_sec_ops = {
 928                 .session_create = aesni_mb_pmd_sec_sess_create,
 929                 .session_update = NULL,
 930                 .session_stats_get = NULL,
 931                 .session_destroy = aesni_mb_pmd_sec_sess_destroy,
 932                 .set_pkt_metadata = NULL,
 933                 .capabilities_get = aesni_mb_pmd_sec_capa_get
 934 };
 935
 936 struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
 937 #endif