drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_string_fns.h>
   8 #include <rte_common.h>
   9 #include <rte_malloc.h>
  10 #include <rte_ether.h>
  11 #include <rte_cryptodev_pmd.h>
  12
  13 #include "aesni_mb_pmd_private.h"
  14
  15
  16 static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
  17         {       /* MD5 HMAC */
  18                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  19                 {.sym = {
  20                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  21                         {.auth = {
  22                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
  23                                 .block_size = 64,
  24                                 .key_size = {
  25                                         .min = 1,
  26                                         .max = 64,
  27                                         .increment = 1
  28                                 },
  29                                 .digest_size = {
  30                                         .min = 1,
  31                                         .max = 16,
  32                                         .increment = 1
  33                                 },
  34                                 .iv_size = { 0 }
  35                         }, }
  36                 }, }
  37         },
  38         {       /* SHA1 HMAC */
  39                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  40                 {.sym = {
  41                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  42                         {.auth = {
  43                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  44                                 .block_size = 64,
  45                                 .key_size = {
  46                                         .min = 1,
  47                                         .max = 65535,
  48                                         .increment = 1
  49                                 },
  50                                 .digest_size = {
  51                                         .min = 1,
  52                                         .max = 20,
  53                                         .increment = 1
  54                                 },
  55                                 .iv_size = { 0 }
  56                         }, }
  57                 }, }
  58         },
  59         {       /* SHA1 */
  60                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  61                 {.sym = {
  62                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  63                         {.auth = {
  64                                 .algo = RTE_CRYPTO_AUTH_SHA1,
  65                                 .block_size = 64,
  66                                 .key_size = {
  67                                         .min = 0,
  68                                         .max = 0,
  69                                         .increment = 0
  70                                 },
  71                                 .digest_size = {
  72                                         .min = 1,
  73                                         .max = 20,
  74                                         .increment = 1
  75                                 },
  76                                 .iv_size = { 0 }
  77                         }, }
  78                 }, }
  79         },
  80         {       /* SHA224 HMAC */
  81                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  82                 {.sym = {
  83                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  84                         {.auth = {
  85                                 .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
  86                                 .block_size = 64,
  87                                 .key_size = {
  88                                         .min = 1,
  89                                         .max = 65535,
  90                                         .increment = 1
  91                                 },
  92                                 .digest_size = {
  93                                         .min = 1,
  94                                         .max = 28,
  95                                         .increment = 1
  96                                 },
  97                                 .iv_size = { 0 }
  98                         }, }
  99                 }, }
 100         },
 101         {       /* SHA224 */
 102                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 103                 {.sym = {
 104                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 105                         {.auth = {
 106                                 .algo = RTE_CRYPTO_AUTH_SHA224,
 107                                 .block_size = 64,
 108                                 .key_size = {
 109                                         .min = 0,
 110                                         .max = 0,
 111                                         .increment = 0
 112                                 },
 113                                 .digest_size = {
 114                                         .min = 1,
 115                                         .max = 28,
 116                                         .increment = 1
 117                                 },
 118                                 .iv_size = { 0 }
 119                         }, }
 120                 }, }
 121         },
 122         {       /* SHA256 HMAC */
 123                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 124                 {.sym = {
 125                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 126                         {.auth = {
 127                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
 128                                 .block_size = 64,
 129                                 .key_size = {
 130                                         .min = 1,
 131                                         .max = 65535,
 132                                         .increment = 1
 133                                 },
 134                                 .digest_size = {
 135                                         .min = 1,
 136                                         .max = 32,
 137                                         .increment = 1
 138                                 },
 139                                 .iv_size = { 0 }
 140                         }, }
 141                 }, }
 142         },
 143         {       /* SHA256 */
 144                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 145                 {.sym = {
 146                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 147                         {.auth = {
 148                                 .algo = RTE_CRYPTO_AUTH_SHA256,
 149                                 .block_size = 64,
 150                                 .key_size = {
 151                                         .min = 0,
 152                                         .max = 0,
 153                                         .increment = 0
 154                                 },
 155                                 .digest_size = {
 156                                         .min = 1,
 157                                         .max = 32,
 158                                         .increment = 1
 159                                 },
 160                                 .iv_size = { 0 }
 161                         }, }
 162                 }, }
 163         },
 164         {       /* SHA384 HMAC */
 165                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 166                 {.sym = {
 167                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 168                         {.auth = {
 169                                 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
 170                                 .block_size = 128,
 171                                 .key_size = {
 172                                         .min = 1,
 173                                         .max = 65535,
 174                                         .increment = 1
 175                                 },
 176                                 .digest_size = {
 177                                         .min = 1,
 178                                         .max = 48,
 179                                         .increment = 1
 180                                 },
 181                                 .iv_size = { 0 }
 182                         }, }
 183                 }, }
 184         },
 185         {       /* SHA384 */
 186                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 187                 {.sym = {
 188                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 189                         {.auth = {
 190                                 .algo = RTE_CRYPTO_AUTH_SHA384,
 191                                 .block_size = 128,
 192                                 .key_size = {
 193                                         .min = 0,
 194                                         .max = 0,
 195                                         .increment = 0
 196                                 },
 197                                 .digest_size = {
 198                                         .min = 1,
 199                                         .max = 48,
 200                                         .increment = 1
 201                                 },
 202                                 .iv_size = { 0 }
 203                         }, }
 204                 }, }
 205         },
 206         {       /* SHA512 HMAC */
 207                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 208                 {.sym = {
 209                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 210                         {.auth = {
 211                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 212                                 .block_size = 128,
 213                                 .key_size = {
 214                                         .min = 1,
 215                                         .max = 65535,
 216                                         .increment = 1
 217                                 },
 218                                 .digest_size = {
 219                                         .min = 1,
 220                                         .max = 64,
 221                                         .increment = 1
 222                                 },
 223                                 .iv_size = { 0 }
 224                         }, }
 225                 }, }
 226         },
 227         {       /* SHA512  */
 228                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 229                 {.sym = {
 230                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 231                         {.auth = {
 232                                 .algo = RTE_CRYPTO_AUTH_SHA512,
 233                                 .block_size = 128,
 234                                 .key_size = {
 235                                         .min = 0,
 236                                         .max = 0,
 237                                         .increment = 0
 238                                 },
 239                                 .digest_size = {
 240                                         .min = 1,
 241                                         .max = 64,
 242                                         .increment = 1
 243                                 },
 244                                 .iv_size = { 0 }
 245                         }, }
 246                 }, }
 247         },
 248         {       /* AES XCBC HMAC */
 249                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 250                 {.sym = {
 251                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 252                         {.auth = {
 253                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 254                                 .block_size = 16,
 255                                 .key_size = {
 256                                         .min = 16,
 257                                         .max = 16,
 258                                         .increment = 0
 259                                 },
 260                                 .digest_size = {
 261                                         .min = 12,
 262                                         .max = 12,
 263                                         .increment = 0
 264                                 },
 265                                 .iv_size = { 0 }
 266                         }, }
 267                 }, }
 268         },
 269         {       /* AES CBC */
 270                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 271                 {.sym = {
 272                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 273                         {.cipher = {
 274                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 275                                 .block_size = 16,
 276                                 .key_size = {
 277                                         .min = 16,
 278                                         .max = 32,
 279                                         .increment = 8
 280                                 },
 281                                 .iv_size = {
 282                                         .min = 16,
 283                                         .max = 16,
 284                                         .increment = 0
 285                                 }
 286                         }, }
 287                 }, }
 288         },
 289         {       /* AES CTR */
 290                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 291                 {.sym = {
 292                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 293                         {.cipher = {
 294                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 295                                 .block_size = 16,
 296                                 .key_size = {
 297                                         .min = 16,
 298                                         .max = 32,
 299                                         .increment = 8
 300                                 },
 301                                 .iv_size = {
 302                                         .min = 12,
 303                                         .max = 16,
 304                                         .increment = 4
 305                                 }
 306                         }, }
 307                 }, }
 308         },
 309         {       /* AES DOCSIS BPI */
 310                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 311                 {.sym = {
 312                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 313                         {.cipher = {
 314                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 315                                 .block_size = 16,
 316                                 .key_size = {
 317                                         .min = 16,
 318 #if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
 319                                         .max = 32,
 320                                         .increment = 16
 321 #else
 322                                         .max = 16,
 323                                         .increment = 0
 324 #endif
 325                                 },
 326                                 .iv_size = {
 327                                         .min = 16,
 328                                         .max = 16,
 329                                         .increment = 0
 330                                 }
 331                         }, }
 332                 }, }
 333         },
 334         {       /* DES CBC */
 335                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 336                 {.sym = {
 337                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 338                         {.cipher = {
 339                                 .algo = RTE_CRYPTO_CIPHER_DES_CBC,
 340                                 .block_size = 8,
 341                                 .key_size = {
 342                                         .min = 8,
 343                                         .max = 8,
 344                                         .increment = 0
 345                                 },
 346                                 .iv_size = {
 347                                         .min = 8,
 348                                         .max = 8,
 349                                         .increment = 0
 350                                 }
 351                         }, }
 352                 }, }
 353         },
 354         {       /*  3DES CBC */
 355                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 356                 {.sym = {
 357                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 358                         {.cipher = {
 359                                 .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
 360                                 .block_size = 8,
 361                                 .key_size = {
 362                                         .min = 8,
 363                                         .max = 24,
 364                                         .increment = 8
 365                                 },
 366                                 .iv_size = {
 367                                         .min = 8,
 368                                         .max = 8,
 369                                         .increment = 0
 370                                 }
 371                         }, }
 372                 }, }
 373         },
 374         {       /* DES DOCSIS BPI */
 375                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 376                 {.sym = {
 377                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 378                         {.cipher = {
 379                                 .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
 380                                 .block_size = 8,
 381                                 .key_size = {
 382                                         .min = 8,
 383                                         .max = 8,
 384                                         .increment = 0
 385                                 },
 386                                 .iv_size = {
 387                                         .min = 8,
 388                                         .max = 8,
 389                                         .increment = 0
 390                                 }
 391                         }, }
 392                 }, }
 393         },
 394         {       /* AES CCM */
 395                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 396                 {.sym = {
 397                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 398                         {.aead = {
 399                                 .algo = RTE_CRYPTO_AEAD_AES_CCM,
 400                                 .block_size = 16,
 401                                 .key_size = {
 402                                         .min = 16,
 403                                         .max = 16,
 404                                         .increment = 0
 405                                 },
 406                                 .digest_size = {
 407                                         .min = 4,
 408                                         .max = 16,
 409                                         .increment = 2
 410                                 },
 411                                 .aad_size = {
 412                                         .min = 0,
 413                                         .max = 46,
 414                                         .increment = 1
 415                                 },
 416                                 .iv_size = {
 417                                         .min = 7,
 418                                         .max = 13,
 419                                         .increment = 1
 420                                 },
 421                         }, }
 422                 }, }
 423         },
 424         {       /* AES CMAC */
 425                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 426                 {.sym = {
 427                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 428                         {.auth = {
 429                                 .algo = RTE_CRYPTO_AUTH_AES_CMAC,
 430                                 .block_size = 16,
 431                                 .key_size = {
 432                                         .min = 16,
 433                                         .max = 16,
 434                                         .increment = 0
 435                                 },
 436                                 .digest_size = {
 437                                         .min = 1,
 438                                         .max = 16,
 439                                         .increment = 1
 440                                 },
 441                                 .iv_size = { 0 }
 442                         }, }
 443                 }, }
 444         },
 445         {       /* AES GCM */
 446                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 447                 {.sym = {
 448                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 449                         {.aead = {
 450                                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
 451                                 .block_size = 16,
 452                                 .key_size = {
 453                                         .min = 16,
 454                                         .max = 32,
 455                                         .increment = 8
 456                                 },
 457                                 .digest_size = {
 458                                         .min = 8,
 459                                         .max = 16,
 460                                         .increment = 4
 461                                 },
 462                                 .aad_size = {
 463                                         .min = 0,
 464                                         .max = 65535,
 465                                         .increment = 1
 466                                 },
 467                                 .iv_size = {
 468                                         .min = 12,
 469                                         .max = 12,
 470                                         .increment = 0
 471                                 }
 472                         }, }
 473                 }, }
 474         },
 475         {       /* AES GMAC (AUTH) */
 476                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 477                 {.sym = {
 478                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 479                         {.auth = {
 480                                 .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 481                                 .block_size = 16,
 482                                 .key_size = {
 483                                         .min = 16,
 484                                         .max = 32,
 485                                         .increment = 8
 486                                 },
 487                                 .digest_size = {
 488                                         .min = 8,
 489                                         .max = 16,
 490                                         .increment = 4
 491                                 },
 492                                 .iv_size = {
 493                                         .min = 12,
 494                                         .max = 12,
 495                                         .increment = 0
 496                                 }
 497                         }, }
 498                 }, }
 499         },
 500 #if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
 501         {       /* AES ECB */
 502                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 503                 {.sym = {
 504                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 505                         {.cipher = {
 506                                 .algo = RTE_CRYPTO_CIPHER_AES_ECB,
 507                                 .block_size = 16,
 508                                 .key_size = {
 509                                         .min = 16,
 510                                         .max = 32,
 511                                         .increment = 8
 512                                 },
 513                                 .iv_size = { 0 }
 514                         }, }
 515                 }, }
 516         },
 517 #endif
 518         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 519 };
 520
 521 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 522 static const struct rte_cryptodev_capabilities
 523                                         aesni_mb_pmd_security_crypto_cap[] = {
 524         {       /* AES DOCSIS BPI */
 525                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 526                 {.sym = {
 527                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 528                         {.cipher = {
 529                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 530                                 .block_size = 16,
 531                                 .key_size = {
 532                                         .min = 16,
 533                                         .max = 32,
 534                                         .increment = 16
 535                                 },
 536                                 .iv_size = {
 537                                         .min = 16,
 538                                         .max = 16,
 539                                         .increment = 0
 540                                 }
 541                         }, }
 542                 }, }
 543         },
 544
 545         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 546 };
 547
 548 static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
 549         {       /* DOCSIS Uplink */
 550                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 551                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 552                 .docsis = {
 553                         .direction = RTE_SECURITY_DOCSIS_UPLINK
 554                 },
 555                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 556         },
 557         {       /* DOCSIS Downlink */
 558                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 559                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 560                 .docsis = {
 561                         .direction = RTE_SECURITY_DOCSIS_DOWNLINK
 562                 },
 563                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 564         },
 565         {
 566                 .action = RTE_SECURITY_ACTION_TYPE_NONE
 567         }
 568 };
 569 #endif
 570
 571 /** Configure device */
 572 static int
 573 aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
 574                 __rte_unused struct rte_cryptodev_config *config)
 575 {
 576         return 0;
 577 }
 578
 579 /** Start device */
 580 static int
 581 aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
 582 {
 583         return 0;
 584 }
 585
 586 /** Stop device */
 587 static void
 588 aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
 589 {
 590 }
 591
 592 /** Close device */
 593 static int
 594 aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
 595 {
 596         return 0;
 597 }
 598
 599
 600 /** Get device statistics */
 601 static void
 602 aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
 603                 struct rte_cryptodev_stats *stats)
 604 {
 605         int qp_id;
 606
 607         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 608                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 609
 610                 stats->enqueued_count += qp->stats.enqueued_count;
 611                 stats->dequeued_count += qp->stats.dequeued_count;
 612
 613                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 614                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 615         }
 616 }
 617
 618 /** Reset device statistics */
 619 static void
 620 aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
 621 {
 622         int qp_id;
 623
 624         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 625                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 626
 627                 memset(&qp->stats, 0, sizeof(qp->stats));
 628         }
 629 }
 630
 631
 632 /** Get device info */
 633 static void
 634 aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
 635                 struct rte_cryptodev_info *dev_info)
 636 {
 637         struct aesni_mb_private *internals = dev->data->dev_private;
 638
 639         if (dev_info != NULL) {
 640                 dev_info->driver_id = dev->driver_id;
 641                 dev_info->feature_flags = dev->feature_flags;
 642                 dev_info->capabilities = aesni_mb_pmd_capabilities;
 643                 dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
 644                 /* No limit of number of sessions */
 645                 dev_info->sym.max_nb_sessions = 0;
 646         }
 647 }
 648
 649 /** Release queue pair */
 650 static int
 651 aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 652 {
 653         struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 654         struct rte_ring *r = NULL;
 655
 656         if (qp != NULL) {
 657                 r = rte_ring_lookup(qp->name);
 658                 if (r)
 659                         rte_ring_free(r);
 660                 if (qp->mb_mgr)
 661                         free_mb_mgr(qp->mb_mgr);
 662                 rte_free(qp);
 663                 dev->data->queue_pairs[qp_id] = NULL;
 664         }
 665         return 0;
 666 }
 667
 668 /** set a unique name for the queue pair based on it's name, dev_id and qp_id */
 669 static int
 670 aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
 671                 struct aesni_mb_qp *qp)
 672 {
 673         unsigned n = snprintf(qp->name, sizeof(qp->name),
 674                         "aesni_mb_pmd_%u_qp_%u",
 675                         dev->data->dev_id, qp->id);
 676
 677         if (n >= sizeof(qp->name))
 678                 return -1;
 679
 680         return 0;
 681 }
 682
 683 /** Create a ring to place processed operations on */
 684 static struct rte_ring *
 685 aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
 686                 unsigned int ring_size, int socket_id)
 687 {
 688         struct rte_ring *r;
 689         char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 690
 691         unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
 692
 693         if (n >= sizeof(ring_name))
 694                 return NULL;
 695
 696         r = rte_ring_lookup(ring_name);
 697         if (r) {
 698                 if (rte_ring_get_size(r) >= ring_size) {
 699                         AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
 700                         ring_name);
 701                         return r;
 702                 }
 703
 704                 AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
 705                         ring_name);
 706                 return NULL;
 707         }
 708
 709         return rte_ring_create(ring_name, ring_size, socket_id,
 710                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 711 }
 712
 713 /** Setup a queue pair */
 714 static int
 715 aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 716                 const struct rte_cryptodev_qp_conf *qp_conf,
 717                 int socket_id)
 718 {
 719         struct aesni_mb_qp *qp = NULL;
 720         struct aesni_mb_private *internals = dev->data->dev_private;
 721         int ret = -1;
 722
 723         /* Free memory prior to re-allocation if needed. */
 724         if (dev->data->queue_pairs[qp_id] != NULL)
 725                 aesni_mb_pmd_qp_release(dev, qp_id);
 726
 727         /* Allocate the queue pair data structure. */
 728         qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
 729                                         RTE_CACHE_LINE_SIZE, socket_id);
 730         if (qp == NULL)
 731                 return -ENOMEM;
 732
 733         qp->id = qp_id;
 734         dev->data->queue_pairs[qp_id] = qp;
 735
 736         if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
 737                 goto qp_setup_cleanup;
 738
 739
 740         qp->mb_mgr = alloc_mb_mgr(0);
 741         if (qp->mb_mgr == NULL) {
 742                 ret = -ENOMEM;
 743                 goto qp_setup_cleanup;
 744         }
 745
 746         switch (internals->vector_mode) {
 747         case RTE_AESNI_MB_SSE:
 748                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
 749                 init_mb_mgr_sse(qp->mb_mgr);
 750                 break;
 751         case RTE_AESNI_MB_AVX:
 752                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
 753                 init_mb_mgr_avx(qp->mb_mgr);
 754                 break;
 755         case RTE_AESNI_MB_AVX2:
 756                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 757                 init_mb_mgr_avx2(qp->mb_mgr);
 758                 break;
 759         case RTE_AESNI_MB_AVX512:
 760                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
 761                 init_mb_mgr_avx512(qp->mb_mgr);
 762                 break;
 763         default:
 764                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
 765                                 internals->vector_mode);
 766                 goto qp_setup_cleanup;
 767         }
 768
 769         qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
 770                         qp_conf->nb_descriptors, socket_id);
 771         if (qp->ingress_queue == NULL) {
 772                 ret = -1;
 773                 goto qp_setup_cleanup;
 774         }
 775
 776         qp->sess_mp = qp_conf->mp_session;
 777         qp->sess_mp_priv = qp_conf->mp_session_private;
 778
 779         memset(&qp->stats, 0, sizeof(qp->stats));
 780
 781         char mp_name[RTE_MEMPOOL_NAMESIZE];
 782
 783         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 784                                 "digest_mp_%u_%u", dev->data->dev_id, qp_id);
 785         return 0;
 786
 787 qp_setup_cleanup:
 788         if (qp) {
 789                 if (qp->mb_mgr)
 790                         free_mb_mgr(qp->mb_mgr);
 791                 rte_free(qp);
 792         }
 793
 794         return ret;
 795 }
 796
 797 /** Returns the size of the aesni multi-buffer session structure */
 798 static unsigned
 799 aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 800 {
 801         return sizeof(struct aesni_mb_session);
 802 }
 803
 804 /** Configure a aesni multi-buffer session from a crypto xform chain */
 805 static int
 806 aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
 807                 struct rte_crypto_sym_xform *xform,
 808                 struct rte_cryptodev_sym_session *sess,
 809                 struct rte_mempool *mempool)
 810 {
 811         void *sess_private_data;
 812         struct aesni_mb_private *internals = dev->data->dev_private;
 813         int ret;
 814
 815         if (unlikely(sess == NULL)) {
 816                 AESNI_MB_LOG(ERR, "invalid session struct");
 817                 return -EINVAL;
 818         }
 819
 820         if (rte_mempool_get(mempool, &sess_private_data)) {
 821                 AESNI_MB_LOG(ERR,
 822                                 "Couldn't get object from session mempool");
 823                 return -ENOMEM;
 824         }
 825
 826         ret = aesni_mb_set_session_parameters(internals->mb_mgr,
 827                         sess_private_data, xform);
 828         if (ret != 0) {
 829                 AESNI_MB_LOG(ERR, "failed configure session parameters");
 830
 831                 /* Return session to mempool */
 832                 rte_mempool_put(mempool, sess_private_data);
 833                 return ret;
 834         }
 835
 836         set_sym_session_private_data(sess, dev->driver_id,
 837                         sess_private_data);
 838
 839         return 0;
 840 }
 841
 842 /** Clear the memory of session so it doesn't leave key material behind */
 843 static void
 844 aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
 845                 struct rte_cryptodev_sym_session *sess)
 846 {
 847         uint8_t index = dev->driver_id;
 848         void *sess_priv = get_sym_session_private_data(sess, index);
 849
 850         /* Zero out the whole structure */
 851         if (sess_priv) {
 852                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
 853                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 854                 set_sym_session_private_data(sess, index, NULL);
 855                 rte_mempool_put(sess_mp, sess_priv);
 856         }
 857 }
 858
 859 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
 860                 .dev_configure          = aesni_mb_pmd_config,
 861                 .dev_start              = aesni_mb_pmd_start,
 862                 .dev_stop               = aesni_mb_pmd_stop,
 863                 .dev_close              = aesni_mb_pmd_close,
 864
 865                 .stats_get              = aesni_mb_pmd_stats_get,
 866                 .stats_reset            = aesni_mb_pmd_stats_reset,
 867
 868                 .dev_infos_get          = aesni_mb_pmd_info_get,
 869
 870                 .queue_pair_setup       = aesni_mb_pmd_qp_setup,
 871                 .queue_pair_release     = aesni_mb_pmd_qp_release,
 872
 873                 .sym_cpu_process        = aesni_mb_cpu_crypto_process_bulk,
 874
 875                 .sym_session_get_size   = aesni_mb_pmd_sym_session_get_size,
 876                 .sym_session_configure  = aesni_mb_pmd_sym_session_configure,
 877                 .sym_session_clear      = aesni_mb_pmd_sym_session_clear
 878 };
 879
 880 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
 881
 882 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 883 /**
 884  * Configure a aesni multi-buffer session from a security session
 885  * configuration
 886  */
 887 static int
 888 aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
 889                 struct rte_security_session *sess,
 890                 struct rte_mempool *mempool)
 891 {
 892         void *sess_private_data;
 893         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
 894         int ret;
 895
 896         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
 897                         conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
 898                 AESNI_MB_LOG(ERR, "Invalid security protocol");
 899                 return -EINVAL;
 900         }
 901
 902         if (rte_mempool_get(mempool, &sess_private_data)) {
 903                 AESNI_MB_LOG(ERR, "Couldn't get object from session mempool");
 904                 return -ENOMEM;
 905         }
 906
 907         ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
 908                         sess_private_data);
 909
 910         if (ret != 0) {
 911                 AESNI_MB_LOG(ERR, "Failed to configure session parameters");
 912
 913                 /* Return session to mempool */
 914                 rte_mempool_put(mempool, sess_private_data);
 915                 return ret;
 916         }
 917
 918         set_sec_session_private_data(sess, sess_private_data);
 919
 920         return ret;
 921 }
 922
 923 /** Clear the memory of session so it doesn't leave key material behind */
 924 static int
 925 aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
 926                 struct rte_security_session *sess)
 927 {
 928         void *sess_priv = get_sec_session_private_data(sess);
 929
 930         if (sess_priv) {
 931                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 932                 memset(sess, 0, sizeof(struct aesni_mb_session));
 933                 set_sec_session_private_data(sess, NULL);
 934                 rte_mempool_put(sess_mp, sess_priv);
 935         }
 936         return 0;
 937 }
 938
 939 /** Get security capabilities for aesni multi-buffer */
 940 static const struct rte_security_capability *
 941 aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
 942 {
 943         return aesni_mb_pmd_security_cap;
 944 }
 945
 946 static struct rte_security_ops aesni_mb_pmd_sec_ops = {
 947                 .session_create = aesni_mb_pmd_sec_sess_create,
 948                 .session_update = NULL,
 949                 .session_stats_get = NULL,
 950                 .session_destroy = aesni_mb_pmd_sec_sess_destroy,
 951                 .set_pkt_metadata = NULL,
 952                 .capabilities_get = aesni_mb_pmd_sec_capa_get
 953 };
 954
 955 struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
 956 #endif