drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_string_fns.h>
   8 #include <rte_common.h>
   9 #include <rte_malloc.h>
  10 #include <rte_ether.h>
  11 #include <rte_cryptodev_pmd.h>
  12
  13 #include "aesni_mb_pmd_private.h"
  14
  15
  16 static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
  17         {       /* MD5 HMAC */
  18                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  19                 {.sym = {
  20                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  21                         {.auth = {
  22                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
  23                                 .block_size = 64,
  24                                 .key_size = {
  25                                         .min = 1,
  26                                         .max = 64,
  27                                         .increment = 1
  28                                 },
  29                                 .digest_size = {
  30                                         .min = 1,
  31                                         .max = 16,
  32                                         .increment = 1
  33                                 },
  34                                 .iv_size = { 0 }
  35                         }, }
  36                 }, }
  37         },
  38         {       /* SHA1 HMAC */
  39                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  40                 {.sym = {
  41                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  42                         {.auth = {
  43                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  44                                 .block_size = 64,
  45                                 .key_size = {
  46                                         .min = 1,
  47                                         .max = 65535,
  48                                         .increment = 1
  49                                 },
  50                                 .digest_size = {
  51                                         .min = 1,
  52                                         .max = 20,
  53                                         .increment = 1
  54                                 },
  55                                 .iv_size = { 0 }
  56                         }, }
  57                 }, }
  58         },
  59         {       /* SHA1 */
  60                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  61                 {.sym = {
  62                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  63                         {.auth = {
  64                                 .algo = RTE_CRYPTO_AUTH_SHA1,
  65                                 .block_size = 64,
  66                                 .key_size = {
  67                                         .min = 0,
  68                                         .max = 0,
  69                                         .increment = 0
  70                                 },
  71                                 .digest_size = {
  72                                         .min = 1,
  73                                         .max = 20,
  74                                         .increment = 1
  75                                 },
  76                                 .iv_size = { 0 }
  77                         }, }
  78                 }, }
  79         },
  80         {       /* SHA224 HMAC */
  81                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  82                 {.sym = {
  83                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  84                         {.auth = {
  85                                 .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
  86                                 .block_size = 64,
  87                                 .key_size = {
  88                                         .min = 1,
  89                                         .max = 65535,
  90                                         .increment = 1
  91                                 },
  92                                 .digest_size = {
  93                                         .min = 1,
  94                                         .max = 28,
  95                                         .increment = 1
  96                                 },
  97                                 .iv_size = { 0 }
  98                         }, }
  99                 }, }
 100         },
 101         {       /* SHA224 */
 102                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 103                 {.sym = {
 104                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 105                         {.auth = {
 106                                 .algo = RTE_CRYPTO_AUTH_SHA224,
 107                                 .block_size = 64,
 108                                 .key_size = {
 109                                         .min = 0,
 110                                         .max = 0,
 111                                         .increment = 0
 112                                 },
 113                                 .digest_size = {
 114                                         .min = 1,
 115                                         .max = 28,
 116                                         .increment = 1
 117                                 },
 118                                 .iv_size = { 0 }
 119                         }, }
 120                 }, }
 121         },
 122         {       /* SHA256 HMAC */
 123                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 124                 {.sym = {
 125                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 126                         {.auth = {
 127                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
 128                                 .block_size = 64,
 129                                 .key_size = {
 130                                         .min = 1,
 131                                         .max = 65535,
 132                                         .increment = 1
 133                                 },
 134                                 .digest_size = {
 135                                         .min = 1,
 136                                         .max = 32,
 137                                         .increment = 1
 138                                 },
 139                                 .iv_size = { 0 }
 140                         }, }
 141                 }, }
 142         },
 143         {       /* SHA256 */
 144                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 145                 {.sym = {
 146                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 147                         {.auth = {
 148                                 .algo = RTE_CRYPTO_AUTH_SHA256,
 149                                 .block_size = 64,
 150                                 .key_size = {
 151                                         .min = 0,
 152                                         .max = 0,
 153                                         .increment = 0
 154                                 },
 155                                 .digest_size = {
 156                                         .min = 1,
 157                                         .max = 32,
 158                                         .increment = 1
 159                                 },
 160                                 .iv_size = { 0 }
 161                         }, }
 162                 }, }
 163         },
 164         {       /* SHA384 HMAC */
 165                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 166                 {.sym = {
 167                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 168                         {.auth = {
 169                                 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
 170                                 .block_size = 128,
 171                                 .key_size = {
 172                                         .min = 1,
 173                                         .max = 65535,
 174                                         .increment = 1
 175                                 },
 176                                 .digest_size = {
 177                                         .min = 1,
 178                                         .max = 48,
 179                                         .increment = 1
 180                                 },
 181                                 .iv_size = { 0 }
 182                         }, }
 183                 }, }
 184         },
 185         {       /* SHA384 */
 186                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 187                 {.sym = {
 188                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 189                         {.auth = {
 190                                 .algo = RTE_CRYPTO_AUTH_SHA384,
 191                                 .block_size = 128,
 192                                 .key_size = {
 193                                         .min = 0,
 194                                         .max = 0,
 195                                         .increment = 0
 196                                 },
 197                                 .digest_size = {
 198                                         .min = 1,
 199                                         .max = 48,
 200                                         .increment = 1
 201                                 },
 202                                 .iv_size = { 0 }
 203                         }, }
 204                 }, }
 205         },
 206         {       /* SHA512 HMAC */
 207                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 208                 {.sym = {
 209                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 210                         {.auth = {
 211                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 212                                 .block_size = 128,
 213                                 .key_size = {
 214                                         .min = 1,
 215                                         .max = 65535,
 216                                         .increment = 1
 217                                 },
 218                                 .digest_size = {
 219                                         .min = 1,
 220                                         .max = 64,
 221                                         .increment = 1
 222                                 },
 223                                 .iv_size = { 0 }
 224                         }, }
 225                 }, }
 226         },
 227         {       /* SHA512  */
 228                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 229                 {.sym = {
 230                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 231                         {.auth = {
 232                                 .algo = RTE_CRYPTO_AUTH_SHA512,
 233                                 .block_size = 128,
 234                                 .key_size = {
 235                                         .min = 0,
 236                                         .max = 0,
 237                                         .increment = 0
 238                                 },
 239                                 .digest_size = {
 240                                         .min = 1,
 241                                         .max = 64,
 242                                         .increment = 1
 243                                 },
 244                                 .iv_size = { 0 }
 245                         }, }
 246                 }, }
 247         },
 248         {       /* AES XCBC HMAC */
 249                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 250                 {.sym = {
 251                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 252                         {.auth = {
 253                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 254                                 .block_size = 16,
 255                                 .key_size = {
 256                                         .min = 16,
 257                                         .max = 16,
 258                                         .increment = 0
 259                                 },
 260                                 .digest_size = {
 261                                         .min = 12,
 262                                         .max = 12,
 263                                         .increment = 0
 264                                 },
 265                                 .iv_size = { 0 }
 266                         }, }
 267                 }, }
 268         },
 269         {       /* AES CBC */
 270                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 271                 {.sym = {
 272                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 273                         {.cipher = {
 274                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 275                                 .block_size = 16,
 276                                 .key_size = {
 277                                         .min = 16,
 278                                         .max = 32,
 279                                         .increment = 8
 280                                 },
 281                                 .iv_size = {
 282                                         .min = 16,
 283                                         .max = 16,
 284                                         .increment = 0
 285                                 }
 286                         }, }
 287                 }, }
 288         },
 289         {       /* AES CTR */
 290                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 291                 {.sym = {
 292                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 293                         {.cipher = {
 294                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 295                                 .block_size = 16,
 296                                 .key_size = {
 297                                         .min = 16,
 298                                         .max = 32,
 299                                         .increment = 8
 300                                 },
 301                                 .iv_size = {
 302                                         .min = 12,
 303                                         .max = 16,
 304                                         .increment = 4
 305                                 }
 306                         }, }
 307                 }, }
 308         },
 309         {       /* AES DOCSIS BPI */
 310                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 311                 {.sym = {
 312                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 313                         {.cipher = {
 314                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 315                                 .block_size = 16,
 316                                 .key_size = {
 317                                         .min = 16,
 318 #if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
 319                                         .max = 32,
 320                                         .increment = 16
 321 #else
 322                                         .max = 16,
 323                                         .increment = 0
 324 #endif
 325                                 },
 326                                 .iv_size = {
 327                                         .min = 16,
 328                                         .max = 16,
 329                                         .increment = 0
 330                                 }
 331                         }, }
 332                 }, }
 333         },
 334         {       /* DES CBC */
 335                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 336                 {.sym = {
 337                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 338                         {.cipher = {
 339                                 .algo = RTE_CRYPTO_CIPHER_DES_CBC,
 340                                 .block_size = 8,
 341                                 .key_size = {
 342                                         .min = 8,
 343                                         .max = 8,
 344                                         .increment = 0
 345                                 },
 346                                 .iv_size = {
 347                                         .min = 8,
 348                                         .max = 8,
 349                                         .increment = 0
 350                                 }
 351                         }, }
 352                 }, }
 353         },
 354         {       /*  3DES CBC */
 355                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 356                 {.sym = {
 357                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 358                         {.cipher = {
 359                                 .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
 360                                 .block_size = 8,
 361                                 .key_size = {
 362                                         .min = 8,
 363                                         .max = 24,
 364                                         .increment = 8
 365                                 },
 366                                 .iv_size = {
 367                                         .min = 8,
 368                                         .max = 8,
 369                                         .increment = 0
 370                                 }
 371                         }, }
 372                 }, }
 373         },
 374         {       /* DES DOCSIS BPI */
 375                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 376                 {.sym = {
 377                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 378                         {.cipher = {
 379                                 .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
 380                                 .block_size = 8,
 381                                 .key_size = {
 382                                         .min = 8,
 383                                         .max = 8,
 384                                         .increment = 0
 385                                 },
 386                                 .iv_size = {
 387                                         .min = 8,
 388                                         .max = 8,
 389                                         .increment = 0
 390                                 }
 391                         }, }
 392                 }, }
 393         },
 394         {       /* AES CCM */
 395                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 396                 {.sym = {
 397                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 398                         {.aead = {
 399                                 .algo = RTE_CRYPTO_AEAD_AES_CCM,
 400                                 .block_size = 16,
 401                                 .key_size = {
 402                                         .min = 16,
 403                                         .max = 16,
 404                                         .increment = 0
 405                                 },
 406                                 .digest_size = {
 407                                         .min = 4,
 408                                         .max = 16,
 409                                         .increment = 2
 410                                 },
 411                                 .aad_size = {
 412                                         .min = 0,
 413                                         .max = 46,
 414                                         .increment = 1
 415                                 },
 416                                 .iv_size = {
 417                                         .min = 7,
 418                                         .max = 13,
 419                                         .increment = 1
 420                                 },
 421                         }, }
 422                 }, }
 423         },
 424         {       /* AES CMAC */
 425                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 426                 {.sym = {
 427                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 428                         {.auth = {
 429                                 .algo = RTE_CRYPTO_AUTH_AES_CMAC,
 430                                 .block_size = 16,
 431                                 .key_size = {
 432                                         .min = 16,
 433                                         .max = 16,
 434                                         .increment = 0
 435                                 },
 436                                 .digest_size = {
 437                                         .min = 1,
 438                                         .max = 16,
 439                                         .increment = 1
 440                                 },
 441                                 .iv_size = { 0 }
 442                         }, }
 443                 }, }
 444         },
 445         {       /* AES GCM */
 446                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 447                 {.sym = {
 448                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 449                         {.aead = {
 450                                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
 451                                 .block_size = 16,
 452                                 .key_size = {
 453                                         .min = 16,
 454                                         .max = 32,
 455                                         .increment = 8
 456                                 },
 457                                 .digest_size = {
 458                                         .min = 1,
 459                                         .max = 16,
 460                                         .increment = 1
 461                                 },
 462                                 .aad_size = {
 463                                         .min = 0,
 464                                         .max = 65535,
 465                                         .increment = 1
 466                                 },
 467                                 .iv_size = {
 468                                         .min = 12,
 469                                         .max = 12,
 470                                         .increment = 0
 471                                 }
 472                         }, }
 473                 }, }
 474         },
 475         {       /* AES GMAC (AUTH) */
 476                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 477                 {.sym = {
 478                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 479                         {.auth = {
 480                                 .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 481                                 .block_size = 16,
 482                                 .key_size = {
 483                                         .min = 16,
 484                                         .max = 32,
 485                                         .increment = 8
 486                                 },
 487                                 .digest_size = {
 488                                         .min = 1,
 489                                         .max = 16,
 490                                         .increment = 1
 491                                 },
 492                                 .iv_size = {
 493                                         .min = 12,
 494                                         .max = 12,
 495                                         .increment = 0
 496                                 }
 497                         }, }
 498                 }, }
 499         },
 500 #if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
 501         {       /* AES ECB */
 502                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 503                 {.sym = {
 504                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 505                         {.cipher = {
 506                                 .algo = RTE_CRYPTO_CIPHER_AES_ECB,
 507                                 .block_size = 16,
 508                                 .key_size = {
 509                                         .min = 16,
 510                                         .max = 32,
 511                                         .increment = 8
 512                                 },
 513                                 .iv_size = { 0 }
 514                         }, }
 515                 }, }
 516         },
 517 #endif
 518 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 519         {       /* ZUC (EIA3) */
 520                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 521                 {.sym = {
 522                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 523                         {.auth = {
 524                                 .algo = RTE_CRYPTO_AUTH_ZUC_EIA3,
 525                                 .block_size = 16,
 526                                 .key_size = {
 527                                         .min = 16,
 528                                         .max = 16,
 529                                         .increment = 0
 530                                 },
 531                                 .digest_size = {
 532                                         .min = 4,
 533                                         .max = 4,
 534                                         .increment = 0
 535                                 },
 536                                 .iv_size = {
 537                                         .min = 16,
 538                                         .max = 16,
 539                                         .increment = 0
 540                                 }
 541                         }, }
 542                 }, }
 543         },
 544         {       /* ZUC (EEA3) */
 545                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 546                 {.sym = {
 547                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 548                         {.cipher = {
 549                                 .algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,
 550                                 .block_size = 16,
 551                                 .key_size = {
 552                                         .min = 16,
 553                                         .max = 16,
 554                                         .increment = 0
 555                                 },
 556                                 .iv_size = {
 557                                         .min = 16,
 558                                         .max = 16,
 559                                         .increment = 0
 560                                 },
 561                         }, }
 562                 }, }
 563         },
 564         {       /* SNOW 3G (UIA2) */
 565                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 566                 {.sym = {
 567                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 568                         {.auth = {
 569                                 .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
 570                                 .block_size = 16,
 571                                 .key_size = {
 572                                         .min = 16,
 573                                         .max = 16,
 574                                         .increment = 0
 575                                 },
 576                                 .digest_size = {
 577                                         .min = 4,
 578                                         .max = 4,
 579                                         .increment = 0
 580                                 },
 581                                 .iv_size = {
 582                                         .min = 16,
 583                                         .max = 16,
 584                                         .increment = 0
 585                                 }
 586                         }, }
 587                 }, }
 588         },
 589         {       /* SNOW 3G (UEA2) */
 590                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 591                 {.sym = {
 592                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 593                         {.cipher = {
 594                                 .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 595                                 .block_size = 16,
 596                                 .key_size = {
 597                                         .min = 16,
 598                                         .max = 16,
 599                                         .increment = 0
 600                                 },
 601                                 .iv_size = {
 602                                         .min = 16,
 603                                         .max = 16,
 604                                         .increment = 0
 605                                 }
 606                         }, }
 607                 }, }
 608         },
 609         {       /* KASUMI (F9) */
 610                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 611                 {.sym = {
 612                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 613                         {.auth = {
 614                                 .algo = RTE_CRYPTO_AUTH_KASUMI_F9,
 615                                 .block_size = 8,
 616                                 .key_size = {
 617                                         .min = 16,
 618                                         .max = 16,
 619                                         .increment = 0
 620                                 },
 621                                 .digest_size = {
 622                                         .min = 4,
 623                                         .max = 4,
 624                                         .increment = 0
 625                                 },
 626                                 .iv_size = { 0 }
 627                         }, }
 628                 }, }
 629         },
 630         {       /* KASUMI (F8) */
 631                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 632                 {.sym = {
 633                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 634                         {.cipher = {
 635                                 .algo = RTE_CRYPTO_CIPHER_KASUMI_F8,
 636                                 .block_size = 8,
 637                                 .key_size = {
 638                                         .min = 16,
 639                                         .max = 16,
 640                                         .increment = 0
 641                                 },
 642                                 .iv_size = {
 643                                         .min = 8,
 644                                         .max = 8,
 645                                         .increment = 0
 646                                 }
 647                         }, }
 648                 }, }
 649         },
 650 #endif
 651         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 652 };
 653
 654 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 655 static const struct rte_cryptodev_capabilities
 656                                         aesni_mb_pmd_security_crypto_cap[] = {
 657         {       /* AES DOCSIS BPI */
 658                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 659                 {.sym = {
 660                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 661                         {.cipher = {
 662                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 663                                 .block_size = 16,
 664                                 .key_size = {
 665                                         .min = 16,
 666                                         .max = 32,
 667                                         .increment = 16
 668                                 },
 669                                 .iv_size = {
 670                                         .min = 16,
 671                                         .max = 16,
 672                                         .increment = 0
 673                                 }
 674                         }, }
 675                 }, }
 676         },
 677
 678         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 679 };
 680
 681 static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
 682         {       /* DOCSIS Uplink */
 683                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 684                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 685                 .docsis = {
 686                         .direction = RTE_SECURITY_DOCSIS_UPLINK
 687                 },
 688                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 689         },
 690         {       /* DOCSIS Downlink */
 691                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 692                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 693                 .docsis = {
 694                         .direction = RTE_SECURITY_DOCSIS_DOWNLINK
 695                 },
 696                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 697         },
 698         {
 699                 .action = RTE_SECURITY_ACTION_TYPE_NONE
 700         }
 701 };
 702 #endif
 703
 704 /** Configure device */
 705 static int
 706 aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
 707                 __rte_unused struct rte_cryptodev_config *config)
 708 {
 709         return 0;
 710 }
 711
 712 /** Start device */
 713 static int
 714 aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
 715 {
 716         return 0;
 717 }
 718
 719 /** Stop device */
 720 static void
 721 aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
 722 {
 723 }
 724
 725 /** Close device */
 726 static int
 727 aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
 728 {
 729         return 0;
 730 }
 731
 732
 733 /** Get device statistics */
 734 static void
 735 aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
 736                 struct rte_cryptodev_stats *stats)
 737 {
 738         int qp_id;
 739
 740         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 741                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 742
 743                 stats->enqueued_count += qp->stats.enqueued_count;
 744                 stats->dequeued_count += qp->stats.dequeued_count;
 745
 746                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 747                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 748         }
 749 }
 750
 751 /** Reset device statistics */
 752 static void
 753 aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
 754 {
 755         int qp_id;
 756
 757         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 758                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 759
 760                 memset(&qp->stats, 0, sizeof(qp->stats));
 761         }
 762 }
 763
 764
 765 /** Get device info */
 766 static void
 767 aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
 768                 struct rte_cryptodev_info *dev_info)
 769 {
 770         struct aesni_mb_private *internals = dev->data->dev_private;
 771
 772         if (dev_info != NULL) {
 773                 dev_info->driver_id = dev->driver_id;
 774                 dev_info->feature_flags = dev->feature_flags;
 775                 dev_info->capabilities = aesni_mb_pmd_capabilities;
 776                 dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
 777                 /* No limit of number of sessions */
 778                 dev_info->sym.max_nb_sessions = 0;
 779         }
 780 }
 781
 782 /** Release queue pair */
 783 static int
 784 aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 785 {
 786         struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 787         struct rte_ring *r = NULL;
 788
 789         if (qp != NULL) {
 790                 r = rte_ring_lookup(qp->name);
 791                 if (r)
 792                         rte_ring_free(r);
 793                 if (qp->mb_mgr)
 794                         free_mb_mgr(qp->mb_mgr);
 795                 rte_free(qp);
 796                 dev->data->queue_pairs[qp_id] = NULL;
 797         }
 798         return 0;
 799 }
 800
 801 /** set a unique name for the queue pair based on it's name, dev_id and qp_id */
 802 static int
 803 aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
 804                 struct aesni_mb_qp *qp)
 805 {
 806         unsigned n = snprintf(qp->name, sizeof(qp->name),
 807                         "aesni_mb_pmd_%u_qp_%u",
 808                         dev->data->dev_id, qp->id);
 809
 810         if (n >= sizeof(qp->name))
 811                 return -1;
 812
 813         return 0;
 814 }
 815
 816 /** Create a ring to place processed operations on */
 817 static struct rte_ring *
 818 aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
 819                 unsigned int ring_size, int socket_id)
 820 {
 821         struct rte_ring *r;
 822         char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 823
 824         unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
 825
 826         if (n >= sizeof(ring_name))
 827                 return NULL;
 828
 829         r = rte_ring_lookup(ring_name);
 830         if (r) {
 831                 if (rte_ring_get_size(r) >= ring_size) {
 832                         AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
 833                         ring_name);
 834                         return r;
 835                 }
 836
 837                 AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
 838                         ring_name);
 839                 return NULL;
 840         }
 841
 842         return rte_ring_create(ring_name, ring_size, socket_id,
 843                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 844 }
 845
 846 /** Setup a queue pair */
 847 static int
 848 aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 849                 const struct rte_cryptodev_qp_conf *qp_conf,
 850                 int socket_id)
 851 {
 852         struct aesni_mb_qp *qp = NULL;
 853         struct aesni_mb_private *internals = dev->data->dev_private;
 854         int ret = -1;
 855
 856         /* Free memory prior to re-allocation if needed. */
 857         if (dev->data->queue_pairs[qp_id] != NULL)
 858                 aesni_mb_pmd_qp_release(dev, qp_id);
 859
 860         /* Allocate the queue pair data structure. */
 861         qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
 862                                         RTE_CACHE_LINE_SIZE, socket_id);
 863         if (qp == NULL)
 864                 return -ENOMEM;
 865
 866         qp->id = qp_id;
 867         dev->data->queue_pairs[qp_id] = qp;
 868
 869         if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
 870                 goto qp_setup_cleanup;
 871
 872
 873         qp->mb_mgr = alloc_mb_mgr(0);
 874         if (qp->mb_mgr == NULL) {
 875                 ret = -ENOMEM;
 876                 goto qp_setup_cleanup;
 877         }
 878
 879         switch (internals->vector_mode) {
 880         case RTE_AESNI_MB_SSE:
 881                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
 882                 init_mb_mgr_sse(qp->mb_mgr);
 883                 break;
 884         case RTE_AESNI_MB_AVX:
 885                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
 886                 init_mb_mgr_avx(qp->mb_mgr);
 887                 break;
 888         case RTE_AESNI_MB_AVX2:
 889                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 890                 init_mb_mgr_avx2(qp->mb_mgr);
 891                 break;
 892         case RTE_AESNI_MB_AVX512:
 893                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
 894                 init_mb_mgr_avx512(qp->mb_mgr);
 895                 break;
 896         default:
 897                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
 898                                 internals->vector_mode);
 899                 goto qp_setup_cleanup;
 900         }
 901
 902         qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
 903                         qp_conf->nb_descriptors, socket_id);
 904         if (qp->ingress_queue == NULL) {
 905                 ret = -1;
 906                 goto qp_setup_cleanup;
 907         }
 908
 909         qp->sess_mp = qp_conf->mp_session;
 910         qp->sess_mp_priv = qp_conf->mp_session_private;
 911
 912         memset(&qp->stats, 0, sizeof(qp->stats));
 913
 914         char mp_name[RTE_MEMPOOL_NAMESIZE];
 915
 916         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 917                                 "digest_mp_%u_%u", dev->data->dev_id, qp_id);
 918         return 0;
 919
 920 qp_setup_cleanup:
 921         if (qp) {
 922                 if (qp->mb_mgr)
 923                         free_mb_mgr(qp->mb_mgr);
 924                 rte_free(qp);
 925         }
 926
 927         return ret;
 928 }
 929
 930 /** Returns the size of the aesni multi-buffer session structure */
 931 static unsigned
 932 aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 933 {
 934         return sizeof(struct aesni_mb_session);
 935 }
 936
 937 /** Configure a aesni multi-buffer session from a crypto xform chain */
 938 static int
 939 aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
 940                 struct rte_crypto_sym_xform *xform,
 941                 struct rte_cryptodev_sym_session *sess,
 942                 struct rte_mempool *mempool)
 943 {
 944         void *sess_private_data;
 945         struct aesni_mb_private *internals = dev->data->dev_private;
 946         int ret;
 947
 948         if (unlikely(sess == NULL)) {
 949                 AESNI_MB_LOG(ERR, "invalid session struct");
 950                 return -EINVAL;
 951         }
 952
 953         if (rte_mempool_get(mempool, &sess_private_data)) {
 954                 AESNI_MB_LOG(ERR,
 955                                 "Couldn't get object from session mempool");
 956                 return -ENOMEM;
 957         }
 958
 959         ret = aesni_mb_set_session_parameters(internals->mb_mgr,
 960                         sess_private_data, xform);
 961         if (ret != 0) {
 962                 AESNI_MB_LOG(ERR, "failed configure session parameters");
 963
 964                 /* Return session to mempool */
 965                 rte_mempool_put(mempool, sess_private_data);
 966                 return ret;
 967         }
 968
 969         set_sym_session_private_data(sess, dev->driver_id,
 970                         sess_private_data);
 971
 972         return 0;
 973 }
 974
 975 /** Clear the memory of session so it doesn't leave key material behind */
 976 static void
 977 aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
 978                 struct rte_cryptodev_sym_session *sess)
 979 {
 980         uint8_t index = dev->driver_id;
 981         void *sess_priv = get_sym_session_private_data(sess, index);
 982
 983         /* Zero out the whole structure */
 984         if (sess_priv) {
 985                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
 986                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 987                 set_sym_session_private_data(sess, index, NULL);
 988                 rte_mempool_put(sess_mp, sess_priv);
 989         }
 990 }
 991
 992 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
 993                 .dev_configure          = aesni_mb_pmd_config,
 994                 .dev_start              = aesni_mb_pmd_start,
 995                 .dev_stop               = aesni_mb_pmd_stop,
 996                 .dev_close              = aesni_mb_pmd_close,
 997
 998                 .stats_get              = aesni_mb_pmd_stats_get,
 999                 .stats_reset            = aesni_mb_pmd_stats_reset,
1000
1001                 .dev_infos_get          = aesni_mb_pmd_info_get,
1002
1003                 .queue_pair_setup       = aesni_mb_pmd_qp_setup,
1004                 .queue_pair_release     = aesni_mb_pmd_qp_release,
1005
1006                 .sym_cpu_process        = aesni_mb_cpu_crypto_process_bulk,
1007
1008                 .sym_session_get_size   = aesni_mb_pmd_sym_session_get_size,
1009                 .sym_session_configure  = aesni_mb_pmd_sym_session_configure,
1010                 .sym_session_clear      = aesni_mb_pmd_sym_session_clear
1011 };
1012
1013 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
1014
1015 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1016 /**
1017  * Configure a aesni multi-buffer session from a security session
1018  * configuration
1019  */
1020 static int
1021 aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
1022                 struct rte_security_session *sess,
1023                 struct rte_mempool *mempool)
1024 {
1025         void *sess_private_data;
1026         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1027         int ret;
1028
1029         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
1030                         conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
1031                 AESNI_MB_LOG(ERR, "Invalid security protocol");
1032                 return -EINVAL;
1033         }
1034
1035         if (rte_mempool_get(mempool, &sess_private_data)) {
1036                 AESNI_MB_LOG(ERR, "Couldn't get object from session mempool");
1037                 return -ENOMEM;
1038         }
1039
1040         ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
1041                         sess_private_data);
1042
1043         if (ret != 0) {
1044                 AESNI_MB_LOG(ERR, "Failed to configure session parameters");
1045
1046                 /* Return session to mempool */
1047                 rte_mempool_put(mempool, sess_private_data);
1048                 return ret;
1049         }
1050
1051         set_sec_session_private_data(sess, sess_private_data);
1052
1053         return ret;
1054 }
1055
1056 /** Clear the memory of session so it doesn't leave key material behind */
1057 static int
1058 aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
1059                 struct rte_security_session *sess)
1060 {
1061         void *sess_priv = get_sec_session_private_data(sess);
1062
1063         if (sess_priv) {
1064                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1065                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
1066                 set_sec_session_private_data(sess, NULL);
1067                 rte_mempool_put(sess_mp, sess_priv);
1068         }
1069         return 0;
1070 }
1071
1072 /** Get security capabilities for aesni multi-buffer */
1073 static const struct rte_security_capability *
1074 aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
1075 {
1076         return aesni_mb_pmd_security_cap;
1077 }
1078
1079 static struct rte_security_ops aesni_mb_pmd_sec_ops = {
1080                 .session_create = aesni_mb_pmd_sec_sess_create,
1081                 .session_update = NULL,
1082                 .session_stats_get = NULL,
1083                 .session_destroy = aesni_mb_pmd_sec_sess_destroy,
1084                 .set_pkt_metadata = NULL,
1085                 .capabilities_get = aesni_mb_pmd_sec_capa_get
1086 };
1087
1088 struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
1089 #endif