drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_string_fns.h>
   8 #include <rte_common.h>
   9 #include <rte_malloc.h>
  10 #include <rte_ether.h>
  11 #include <rte_cryptodev_pmd.h>
  12
  13 #include "aesni_mb_pmd_private.h"
  14
  15
  16 static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
  17         {       /* MD5 HMAC */
  18                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  19                 {.sym = {
  20                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  21                         {.auth = {
  22                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
  23                                 .block_size = 64,
  24                                 .key_size = {
  25                                         .min = 1,
  26                                         .max = 64,
  27                                         .increment = 1
  28                                 },
  29                                 .digest_size = {
  30                                         .min = 1,
  31                                         .max = 16,
  32                                         .increment = 1
  33                                 },
  34                                 .iv_size = { 0 }
  35                         }, }
  36                 }, }
  37         },
  38         {       /* SHA1 HMAC */
  39                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  40                 {.sym = {
  41                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  42                         {.auth = {
  43                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  44                                 .block_size = 64,
  45                                 .key_size = {
  46                                         .min = 1,
  47                                         .max = 65535,
  48                                         .increment = 1
  49                                 },
  50                                 .digest_size = {
  51                                         .min = 1,
  52                                         .max = 20,
  53                                         .increment = 1
  54                                 },
  55                                 .iv_size = { 0 }
  56                         }, }
  57                 }, }
  58         },
  59         {       /* SHA1 */
  60                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  61                 {.sym = {
  62                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  63                         {.auth = {
  64                                 .algo = RTE_CRYPTO_AUTH_SHA1,
  65                                 .block_size = 64,
  66                                 .key_size = {
  67                                         .min = 0,
  68                                         .max = 0,
  69                                         .increment = 0
  70                                 },
  71                                 .digest_size = {
  72                                         .min = 1,
  73                                         .max = 20,
  74                                         .increment = 1
  75                                 },
  76                                 .iv_size = { 0 }
  77                         }, }
  78                 }, }
  79         },
  80         {       /* SHA224 HMAC */
  81                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  82                 {.sym = {
  83                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  84                         {.auth = {
  85                                 .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
  86                                 .block_size = 64,
  87                                 .key_size = {
  88                                         .min = 1,
  89                                         .max = 65535,
  90                                         .increment = 1
  91                                 },
  92                                 .digest_size = {
  93                                         .min = 1,
  94                                         .max = 28,
  95                                         .increment = 1
  96                                 },
  97                                 .iv_size = { 0 }
  98                         }, }
  99                 }, }
 100         },
 101         {       /* SHA224 */
 102                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 103                 {.sym = {
 104                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 105                         {.auth = {
 106                                 .algo = RTE_CRYPTO_AUTH_SHA224,
 107                                 .block_size = 64,
 108                                 .key_size = {
 109                                         .min = 0,
 110                                         .max = 0,
 111                                         .increment = 0
 112                                 },
 113                                 .digest_size = {
 114                                         .min = 1,
 115                                         .max = 28,
 116                                         .increment = 1
 117                                 },
 118                                 .iv_size = { 0 }
 119                         }, }
 120                 }, }
 121         },
 122         {       /* SHA256 HMAC */
 123                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 124                 {.sym = {
 125                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 126                         {.auth = {
 127                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
 128                                 .block_size = 64,
 129                                 .key_size = {
 130                                         .min = 1,
 131                                         .max = 65535,
 132                                         .increment = 1
 133                                 },
 134                                 .digest_size = {
 135                                         .min = 1,
 136                                         .max = 32,
 137                                         .increment = 1
 138                                 },
 139                                 .iv_size = { 0 }
 140                         }, }
 141                 }, }
 142         },
 143         {       /* SHA256 */
 144                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 145                 {.sym = {
 146                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 147                         {.auth = {
 148                                 .algo = RTE_CRYPTO_AUTH_SHA256,
 149                                 .block_size = 64,
 150                                 .key_size = {
 151                                         .min = 0,
 152                                         .max = 0,
 153                                         .increment = 0
 154                                 },
 155                                 .digest_size = {
 156                                         .min = 1,
 157                                         .max = 32,
 158                                         .increment = 1
 159                                 },
 160                                 .iv_size = { 0 }
 161                         }, }
 162                 }, }
 163         },
 164         {       /* SHA384 HMAC */
 165                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 166                 {.sym = {
 167                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 168                         {.auth = {
 169                                 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
 170                                 .block_size = 128,
 171                                 .key_size = {
 172                                         .min = 1,
 173                                         .max = 65535,
 174                                         .increment = 1
 175                                 },
 176                                 .digest_size = {
 177                                         .min = 1,
 178                                         .max = 48,
 179                                         .increment = 1
 180                                 },
 181                                 .iv_size = { 0 }
 182                         }, }
 183                 }, }
 184         },
 185         {       /* SHA384 */
 186                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 187                 {.sym = {
 188                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 189                         {.auth = {
 190                                 .algo = RTE_CRYPTO_AUTH_SHA384,
 191                                 .block_size = 128,
 192                                 .key_size = {
 193                                         .min = 0,
 194                                         .max = 0,
 195                                         .increment = 0
 196                                 },
 197                                 .digest_size = {
 198                                         .min = 1,
 199                                         .max = 48,
 200                                         .increment = 1
 201                                 },
 202                                 .iv_size = { 0 }
 203                         }, }
 204                 }, }
 205         },
 206         {       /* SHA512 HMAC */
 207                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 208                 {.sym = {
 209                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 210                         {.auth = {
 211                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 212                                 .block_size = 128,
 213                                 .key_size = {
 214                                         .min = 1,
 215                                         .max = 65535,
 216                                         .increment = 1
 217                                 },
 218                                 .digest_size = {
 219                                         .min = 1,
 220                                         .max = 64,
 221                                         .increment = 1
 222                                 },
 223                                 .iv_size = { 0 }
 224                         }, }
 225                 }, }
 226         },
 227         {       /* SHA512  */
 228                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 229                 {.sym = {
 230                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 231                         {.auth = {
 232                                 .algo = RTE_CRYPTO_AUTH_SHA512,
 233                                 .block_size = 128,
 234                                 .key_size = {
 235                                         .min = 0,
 236                                         .max = 0,
 237                                         .increment = 0
 238                                 },
 239                                 .digest_size = {
 240                                         .min = 1,
 241                                         .max = 64,
 242                                         .increment = 1
 243                                 },
 244                                 .iv_size = { 0 }
 245                         }, }
 246                 }, }
 247         },
 248         {       /* AES XCBC HMAC */
 249                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 250                 {.sym = {
 251                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 252                         {.auth = {
 253                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 254                                 .block_size = 16,
 255                                 .key_size = {
 256                                         .min = 16,
 257                                         .max = 16,
 258                                         .increment = 0
 259                                 },
 260                                 .digest_size = {
 261                                         .min = 12,
 262                                         .max = 12,
 263                                         .increment = 0
 264                                 },
 265                                 .iv_size = { 0 }
 266                         }, }
 267                 }, }
 268         },
 269         {       /* AES CBC */
 270                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 271                 {.sym = {
 272                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 273                         {.cipher = {
 274                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 275                                 .block_size = 16,
 276                                 .key_size = {
 277                                         .min = 16,
 278                                         .max = 32,
 279                                         .increment = 8
 280                                 },
 281                                 .iv_size = {
 282                                         .min = 16,
 283                                         .max = 16,
 284                                         .increment = 0
 285                                 }
 286                         }, }
 287                 }, }
 288         },
 289         {       /* AES CTR */
 290                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 291                 {.sym = {
 292                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 293                         {.cipher = {
 294                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 295                                 .block_size = 16,
 296                                 .key_size = {
 297                                         .min = 16,
 298                                         .max = 32,
 299                                         .increment = 8
 300                                 },
 301                                 .iv_size = {
 302                                         .min = 12,
 303                                         .max = 16,
 304                                         .increment = 4
 305                                 }
 306                         }, }
 307                 }, }
 308         },
 309         {       /* AES DOCSIS BPI */
 310                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 311                 {.sym = {
 312                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 313                         {.cipher = {
 314                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 315                                 .block_size = 16,
 316                                 .key_size = {
 317                                         .min = 16,
 318 #if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
 319                                         .max = 32,
 320                                         .increment = 16
 321 #else
 322                                         .max = 16,
 323                                         .increment = 0
 324 #endif
 325                                 },
 326                                 .iv_size = {
 327                                         .min = 16,
 328                                         .max = 16,
 329                                         .increment = 0
 330                                 }
 331                         }, }
 332                 }, }
 333         },
 334         {       /* DES CBC */
 335                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 336                 {.sym = {
 337                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 338                         {.cipher = {
 339                                 .algo = RTE_CRYPTO_CIPHER_DES_CBC,
 340                                 .block_size = 8,
 341                                 .key_size = {
 342                                         .min = 8,
 343                                         .max = 8,
 344                                         .increment = 0
 345                                 },
 346                                 .iv_size = {
 347                                         .min = 8,
 348                                         .max = 8,
 349                                         .increment = 0
 350                                 }
 351                         }, }
 352                 }, }
 353         },
 354         {       /*  3DES CBC */
 355                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 356                 {.sym = {
 357                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 358                         {.cipher = {
 359                                 .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
 360                                 .block_size = 8,
 361                                 .key_size = {
 362                                         .min = 8,
 363                                         .max = 24,
 364                                         .increment = 8
 365                                 },
 366                                 .iv_size = {
 367                                         .min = 8,
 368                                         .max = 8,
 369                                         .increment = 0
 370                                 }
 371                         }, }
 372                 }, }
 373         },
 374         {       /* DES DOCSIS BPI */
 375                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 376                 {.sym = {
 377                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 378                         {.cipher = {
 379                                 .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
 380                                 .block_size = 8,
 381                                 .key_size = {
 382                                         .min = 8,
 383                                         .max = 8,
 384                                         .increment = 0
 385                                 },
 386                                 .iv_size = {
 387                                         .min = 8,
 388                                         .max = 8,
 389                                         .increment = 0
 390                                 }
 391                         }, }
 392                 }, }
 393         },
 394         {       /* AES CCM */
 395                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 396                 {.sym = {
 397                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 398                         {.aead = {
 399                                 .algo = RTE_CRYPTO_AEAD_AES_CCM,
 400                                 .block_size = 16,
 401                                 .key_size = {
 402                                         .min = 16,
 403                                         .max = 16,
 404                                         .increment = 0
 405                                 },
 406                                 .digest_size = {
 407                                         .min = 4,
 408                                         .max = 16,
 409                                         .increment = 2
 410                                 },
 411                                 .aad_size = {
 412                                         .min = 0,
 413                                         .max = 46,
 414                                         .increment = 1
 415                                 },
 416                                 .iv_size = {
 417                                         .min = 7,
 418                                         .max = 13,
 419                                         .increment = 1
 420                                 },
 421                         }, }
 422                 }, }
 423         },
 424         {       /* AES CMAC */
 425                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 426                 {.sym = {
 427                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 428                         {.auth = {
 429                                 .algo = RTE_CRYPTO_AUTH_AES_CMAC,
 430                                 .block_size = 16,
 431                                 .key_size = {
 432                                         .min = 16,
 433                                         .max = 16,
 434                                         .increment = 0
 435                                 },
 436                                 .digest_size = {
 437                                         .min = 1,
 438                                         .max = 16,
 439                                         .increment = 1
 440                                 },
 441                                 .iv_size = { 0 }
 442                         }, }
 443                 }, }
 444         },
 445         {       /* AES GCM */
 446                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 447                 {.sym = {
 448                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 449                         {.aead = {
 450                                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
 451                                 .block_size = 16,
 452                                 .key_size = {
 453                                         .min = 16,
 454                                         .max = 32,
 455                                         .increment = 8
 456                                 },
 457                                 .digest_size = {
 458                                         .min = 1,
 459                                         .max = 16,
 460                                         .increment = 1
 461                                 },
 462                                 .aad_size = {
 463                                         .min = 0,
 464                                         .max = 65535,
 465                                         .increment = 1
 466                                 },
 467                                 .iv_size = {
 468                                         .min = 12,
 469                                         .max = 12,
 470                                         .increment = 0
 471                                 }
 472                         }, }
 473                 }, }
 474         },
 475         {       /* AES GMAC (AUTH) */
 476                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 477                 {.sym = {
 478                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 479                         {.auth = {
 480                                 .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 481                                 .block_size = 16,
 482                                 .key_size = {
 483                                         .min = 16,
 484                                         .max = 32,
 485                                         .increment = 8
 486                                 },
 487                                 .digest_size = {
 488                                         .min = 1,
 489                                         .max = 16,
 490                                         .increment = 1
 491                                 },
 492                                 .iv_size = {
 493                                         .min = 12,
 494                                         .max = 12,
 495                                         .increment = 0
 496                                 }
 497                         }, }
 498                 }, }
 499         },
 500 #if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
 501         {       /* AES ECB */
 502                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 503                 {.sym = {
 504                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 505                         {.cipher = {
 506                                 .algo = RTE_CRYPTO_CIPHER_AES_ECB,
 507                                 .block_size = 16,
 508                                 .key_size = {
 509                                         .min = 16,
 510                                         .max = 32,
 511                                         .increment = 8
 512                                 },
 513                                 .iv_size = { 0 }
 514                         }, }
 515                 }, }
 516         },
 517 #endif
 518 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 519         {       /* ZUC (EIA3) */
 520                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 521                 {.sym = {
 522                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 523                         {.auth = {
 524                                 .algo = RTE_CRYPTO_AUTH_ZUC_EIA3,
 525                                 .block_size = 16,
 526                                 .key_size = {
 527                                         .min = 16,
 528                                         .max = 16,
 529                                         .increment = 0
 530                                 },
 531                                 .digest_size = {
 532                                         .min = 4,
 533                                         .max = 4,
 534                                         .increment = 0
 535                                 },
 536                                 .iv_size = {
 537                                         .min = 16,
 538                                         .max = 16,
 539                                         .increment = 0
 540                                 }
 541                         }, }
 542                 }, }
 543         },
 544         {       /* ZUC (EEA3) */
 545                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 546                 {.sym = {
 547                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 548                         {.cipher = {
 549                                 .algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,
 550                                 .block_size = 16,
 551                                 .key_size = {
 552                                         .min = 16,
 553                                         .max = 16,
 554                                         .increment = 0
 555                                 },
 556                                 .iv_size = {
 557                                         .min = 16,
 558                                         .max = 16,
 559                                         .increment = 0
 560                                 },
 561                         }, }
 562                 }, }
 563         },
 564         {       /* SNOW 3G (UIA2) */
 565                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 566                 {.sym = {
 567                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 568                         {.auth = {
 569                                 .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
 570                                 .block_size = 16,
 571                                 .key_size = {
 572                                         .min = 16,
 573                                         .max = 16,
 574                                         .increment = 0
 575                                 },
 576                                 .digest_size = {
 577                                         .min = 4,
 578                                         .max = 4,
 579                                         .increment = 0
 580                                 },
 581                                 .iv_size = {
 582                                         .min = 16,
 583                                         .max = 16,
 584                                         .increment = 0
 585                                 }
 586                         }, }
 587                 }, }
 588         },
 589         {       /* SNOW 3G (UEA2) */
 590                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 591                 {.sym = {
 592                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 593                         {.cipher = {
 594                                 .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 595                                 .block_size = 16,
 596                                 .key_size = {
 597                                         .min = 16,
 598                                         .max = 16,
 599                                         .increment = 0
 600                                 },
 601                                 .iv_size = {
 602                                         .min = 16,
 603                                         .max = 16,
 604                                         .increment = 0
 605                                 }
 606                         }, }
 607                 }, }
 608         },
 609         {       /* KASUMI (F9) */
 610                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 611                 {.sym = {
 612                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 613                         {.auth = {
 614                                 .algo = RTE_CRYPTO_AUTH_KASUMI_F9,
 615                                 .block_size = 8,
 616                                 .key_size = {
 617                                         .min = 16,
 618                                         .max = 16,
 619                                         .increment = 0
 620                                 },
 621                                 .digest_size = {
 622                                         .min = 4,
 623                                         .max = 4,
 624                                         .increment = 0
 625                                 },
 626                                 .iv_size = { 0 }
 627                         }, }
 628                 }, }
 629         },
 630         {       /* KASUMI (F8) */
 631                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 632                 {.sym = {
 633                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 634                         {.cipher = {
 635                                 .algo = RTE_CRYPTO_CIPHER_KASUMI_F8,
 636                                 .block_size = 8,
 637                                 .key_size = {
 638                                         .min = 16,
 639                                         .max = 16,
 640                                         .increment = 0
 641                                 },
 642                                 .iv_size = {
 643                                         .min = 8,
 644                                         .max = 8,
 645                                         .increment = 0
 646                                 }
 647                         }, }
 648                 }, }
 649         },
 650 #endif
 651 #if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
 652         {       /* CHACHA20-POLY1305 */
 653                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 654                 {.sym = {
 655                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 656                         {.aead = {
 657                                 .algo = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,
 658                                 .block_size = 64,
 659                                 .key_size = {
 660                                         .min = 32,
 661                                         .max = 32,
 662                                         .increment = 0
 663                                 },
 664                                 .digest_size = {
 665                                         .min = 16,
 666                                         .max = 16,
 667                                         .increment = 0
 668                                 },
 669                                 .aad_size = {
 670                                         .min = 0,
 671                                         .max = 240,
 672                                         .increment = 1
 673                                 },
 674                                 .iv_size = {
 675                                         .min = 12,
 676                                         .max = 12,
 677                                         .increment = 0
 678                                 },
 679                         }, }
 680                 }, }
 681         },
 682 #endif
 683         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 684 };
 685
 686 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 687 static const struct rte_cryptodev_capabilities
 688                                         aesni_mb_pmd_security_crypto_cap[] = {
 689         {       /* AES DOCSIS BPI */
 690                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 691                 {.sym = {
 692                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 693                         {.cipher = {
 694                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 695                                 .block_size = 16,
 696                                 .key_size = {
 697                                         .min = 16,
 698                                         .max = 32,
 699                                         .increment = 16
 700                                 },
 701                                 .iv_size = {
 702                                         .min = 16,
 703                                         .max = 16,
 704                                         .increment = 0
 705                                 }
 706                         }, }
 707                 }, }
 708         },
 709
 710         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 711 };
 712
 713 static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
 714         {       /* DOCSIS Uplink */
 715                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 716                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 717                 .docsis = {
 718                         .direction = RTE_SECURITY_DOCSIS_UPLINK
 719                 },
 720                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 721         },
 722         {       /* DOCSIS Downlink */
 723                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 724                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 725                 .docsis = {
 726                         .direction = RTE_SECURITY_DOCSIS_DOWNLINK
 727                 },
 728                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 729         },
 730         {
 731                 .action = RTE_SECURITY_ACTION_TYPE_NONE
 732         }
 733 };
 734 #endif
 735
 736 /** Configure device */
 737 static int
 738 aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
 739                 __rte_unused struct rte_cryptodev_config *config)
 740 {
 741         return 0;
 742 }
 743
 744 /** Start device */
 745 static int
 746 aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
 747 {
 748         return 0;
 749 }
 750
 751 /** Stop device */
 752 static void
 753 aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
 754 {
 755 }
 756
 757 /** Close device */
 758 static int
 759 aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
 760 {
 761         return 0;
 762 }
 763
 764
 765 /** Get device statistics */
 766 static void
 767 aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
 768                 struct rte_cryptodev_stats *stats)
 769 {
 770         int qp_id;
 771
 772         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 773                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 774
 775                 stats->enqueued_count += qp->stats.enqueued_count;
 776                 stats->dequeued_count += qp->stats.dequeued_count;
 777
 778                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 779                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 780         }
 781 }
 782
 783 /** Reset device statistics */
 784 static void
 785 aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
 786 {
 787         int qp_id;
 788
 789         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 790                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 791
 792                 memset(&qp->stats, 0, sizeof(qp->stats));
 793         }
 794 }
 795
 796
 797 /** Get device info */
 798 static void
 799 aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
 800                 struct rte_cryptodev_info *dev_info)
 801 {
 802         struct aesni_mb_private *internals = dev->data->dev_private;
 803
 804         if (dev_info != NULL) {
 805                 dev_info->driver_id = dev->driver_id;
 806                 dev_info->feature_flags = dev->feature_flags;
 807                 dev_info->capabilities = aesni_mb_pmd_capabilities;
 808                 dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
 809                 /* No limit of number of sessions */
 810                 dev_info->sym.max_nb_sessions = 0;
 811         }
 812 }
 813
 814 /** Release queue pair */
 815 static int
 816 aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 817 {
 818         struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 819         struct rte_ring *r = NULL;
 820
 821         if (qp != NULL) {
 822                 r = rte_ring_lookup(qp->name);
 823                 if (r)
 824                         rte_ring_free(r);
 825                 if (qp->mb_mgr)
 826                         free_mb_mgr(qp->mb_mgr);
 827                 rte_free(qp);
 828                 dev->data->queue_pairs[qp_id] = NULL;
 829         }
 830         return 0;
 831 }
 832
 833 /** set a unique name for the queue pair based on it's name, dev_id and qp_id */
 834 static int
 835 aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
 836                 struct aesni_mb_qp *qp)
 837 {
 838         unsigned n = snprintf(qp->name, sizeof(qp->name),
 839                         "aesni_mb_pmd_%u_qp_%u",
 840                         dev->data->dev_id, qp->id);
 841
 842         if (n >= sizeof(qp->name))
 843                 return -1;
 844
 845         return 0;
 846 }
 847
 848 /** Create a ring to place processed operations on */
 849 static struct rte_ring *
 850 aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
 851                 unsigned int ring_size, int socket_id)
 852 {
 853         struct rte_ring *r;
 854         char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 855
 856         unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
 857
 858         if (n >= sizeof(ring_name))
 859                 return NULL;
 860
 861         r = rte_ring_lookup(ring_name);
 862         if (r) {
 863                 if (rte_ring_get_size(r) >= ring_size) {
 864                         AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
 865                         ring_name);
 866                         return r;
 867                 }
 868
 869                 AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
 870                         ring_name);
 871                 return NULL;
 872         }
 873
 874         return rte_ring_create(ring_name, ring_size, socket_id,
 875                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 876 }
 877
 878 /** Setup a queue pair */
 879 static int
 880 aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 881                 const struct rte_cryptodev_qp_conf *qp_conf,
 882                 int socket_id)
 883 {
 884         struct aesni_mb_qp *qp = NULL;
 885         struct aesni_mb_private *internals = dev->data->dev_private;
 886         int ret = -1;
 887
 888         /* Free memory prior to re-allocation if needed. */
 889         if (dev->data->queue_pairs[qp_id] != NULL)
 890                 aesni_mb_pmd_qp_release(dev, qp_id);
 891
 892         /* Allocate the queue pair data structure. */
 893         qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
 894                                         RTE_CACHE_LINE_SIZE, socket_id);
 895         if (qp == NULL)
 896                 return -ENOMEM;
 897
 898         qp->id = qp_id;
 899         dev->data->queue_pairs[qp_id] = qp;
 900
 901         if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
 902                 goto qp_setup_cleanup;
 903
 904
 905         qp->mb_mgr = alloc_mb_mgr(0);
 906         if (qp->mb_mgr == NULL) {
 907                 ret = -ENOMEM;
 908                 goto qp_setup_cleanup;
 909         }
 910
 911         switch (internals->vector_mode) {
 912         case RTE_AESNI_MB_SSE:
 913                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
 914                 init_mb_mgr_sse(qp->mb_mgr);
 915                 break;
 916         case RTE_AESNI_MB_AVX:
 917                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
 918                 init_mb_mgr_avx(qp->mb_mgr);
 919                 break;
 920         case RTE_AESNI_MB_AVX2:
 921                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 922                 init_mb_mgr_avx2(qp->mb_mgr);
 923                 break;
 924         case RTE_AESNI_MB_AVX512:
 925                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
 926                 init_mb_mgr_avx512(qp->mb_mgr);
 927                 break;
 928         default:
 929                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
 930                                 internals->vector_mode);
 931                 goto qp_setup_cleanup;
 932         }
 933
 934         qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
 935                         qp_conf->nb_descriptors, socket_id);
 936         if (qp->ingress_queue == NULL) {
 937                 ret = -1;
 938                 goto qp_setup_cleanup;
 939         }
 940
 941         qp->sess_mp = qp_conf->mp_session;
 942         qp->sess_mp_priv = qp_conf->mp_session_private;
 943
 944         memset(&qp->stats, 0, sizeof(qp->stats));
 945
 946         char mp_name[RTE_MEMPOOL_NAMESIZE];
 947
 948         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 949                                 "digest_mp_%u_%u", dev->data->dev_id, qp_id);
 950         return 0;
 951
 952 qp_setup_cleanup:
 953         if (qp) {
 954                 if (qp->mb_mgr)
 955                         free_mb_mgr(qp->mb_mgr);
 956                 rte_free(qp);
 957         }
 958
 959         return ret;
 960 }
 961
 962 /** Returns the size of the aesni multi-buffer session structure */
 963 static unsigned
 964 aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 965 {
 966         return sizeof(struct aesni_mb_session);
 967 }
 968
 969 /** Configure a aesni multi-buffer session from a crypto xform chain */
 970 static int
 971 aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
 972                 struct rte_crypto_sym_xform *xform,
 973                 struct rte_cryptodev_sym_session *sess,
 974                 struct rte_mempool *mempool)
 975 {
 976         void *sess_private_data;
 977         struct aesni_mb_private *internals = dev->data->dev_private;
 978         int ret;
 979
 980         if (unlikely(sess == NULL)) {
 981                 AESNI_MB_LOG(ERR, "invalid session struct");
 982                 return -EINVAL;
 983         }
 984
 985         if (rte_mempool_get(mempool, &sess_private_data)) {
 986                 AESNI_MB_LOG(ERR,
 987                                 "Couldn't get object from session mempool");
 988                 return -ENOMEM;
 989         }
 990
 991         ret = aesni_mb_set_session_parameters(internals->mb_mgr,
 992                         sess_private_data, xform);
 993         if (ret != 0) {
 994                 AESNI_MB_LOG(ERR, "failed configure session parameters");
 995
 996                 /* Return session to mempool */
 997                 rte_mempool_put(mempool, sess_private_data);
 998                 return ret;
 999         }
1000
1001         set_sym_session_private_data(sess, dev->driver_id,
1002                         sess_private_data);
1003
1004         return 0;
1005 }
1006
1007 /** Clear the memory of session so it doesn't leave key material behind */
1008 static void
1009 aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
1010                 struct rte_cryptodev_sym_session *sess)
1011 {
1012         uint8_t index = dev->driver_id;
1013         void *sess_priv = get_sym_session_private_data(sess, index);
1014
1015         /* Zero out the whole structure */
1016         if (sess_priv) {
1017                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
1018                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1019                 set_sym_session_private_data(sess, index, NULL);
1020                 rte_mempool_put(sess_mp, sess_priv);
1021         }
1022 }
1023
1024 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
1025                 .dev_configure          = aesni_mb_pmd_config,
1026                 .dev_start              = aesni_mb_pmd_start,
1027                 .dev_stop               = aesni_mb_pmd_stop,
1028                 .dev_close              = aesni_mb_pmd_close,
1029
1030                 .stats_get              = aesni_mb_pmd_stats_get,
1031                 .stats_reset            = aesni_mb_pmd_stats_reset,
1032
1033                 .dev_infos_get          = aesni_mb_pmd_info_get,
1034
1035                 .queue_pair_setup       = aesni_mb_pmd_qp_setup,
1036                 .queue_pair_release     = aesni_mb_pmd_qp_release,
1037
1038                 .sym_cpu_process        = aesni_mb_cpu_crypto_process_bulk,
1039
1040                 .sym_session_get_size   = aesni_mb_pmd_sym_session_get_size,
1041                 .sym_session_configure  = aesni_mb_pmd_sym_session_configure,
1042                 .sym_session_clear      = aesni_mb_pmd_sym_session_clear
1043 };
1044
1045 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
1046
1047 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1048 /**
1049  * Configure a aesni multi-buffer session from a security session
1050  * configuration
1051  */
1052 static int
1053 aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
1054                 struct rte_security_session *sess,
1055                 struct rte_mempool *mempool)
1056 {
1057         void *sess_private_data;
1058         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1059         int ret;
1060
1061         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
1062                         conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
1063                 AESNI_MB_LOG(ERR, "Invalid security protocol");
1064                 return -EINVAL;
1065         }
1066
1067         if (rte_mempool_get(mempool, &sess_private_data)) {
1068                 AESNI_MB_LOG(ERR, "Couldn't get object from session mempool");
1069                 return -ENOMEM;
1070         }
1071
1072         ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
1073                         sess_private_data);
1074
1075         if (ret != 0) {
1076                 AESNI_MB_LOG(ERR, "Failed to configure session parameters");
1077
1078                 /* Return session to mempool */
1079                 rte_mempool_put(mempool, sess_private_data);
1080                 return ret;
1081         }
1082
1083         set_sec_session_private_data(sess, sess_private_data);
1084
1085         return ret;
1086 }
1087
1088 /** Clear the memory of session so it doesn't leave key material behind */
1089 static int
1090 aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
1091                 struct rte_security_session *sess)
1092 {
1093         void *sess_priv = get_sec_session_private_data(sess);
1094
1095         if (sess_priv) {
1096                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1097                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
1098                 set_sec_session_private_data(sess, NULL);
1099                 rte_mempool_put(sess_mp, sess_priv);
1100         }
1101         return 0;
1102 }
1103
1104 /** Get security capabilities for aesni multi-buffer */
1105 static const struct rte_security_capability *
1106 aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
1107 {
1108         return aesni_mb_pmd_security_cap;
1109 }
1110
1111 static struct rte_security_ops aesni_mb_pmd_sec_ops = {
1112                 .session_create = aesni_mb_pmd_sec_sess_create,
1113                 .session_update = NULL,
1114                 .session_stats_get = NULL,
1115                 .session_destroy = aesni_mb_pmd_sec_sess_destroy,
1116                 .set_pkt_metadata = NULL,
1117                 .capabilities_get = aesni_mb_pmd_sec_capa_get
1118 };
1119
1120 struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
1121 #endif