drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_string_fns.h>
   8 #include <rte_common.h>
   9 #include <rte_malloc.h>
  10 #include <rte_ether.h>
  11 #include <cryptodev_pmd.h>
  12
  13 #include "aesni_mb_pmd_private.h"
  14
  15
  16 static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
  17         {       /* MD5 HMAC */
  18                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  19                 {.sym = {
  20                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  21                         {.auth = {
  22                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
  23                                 .block_size = 64,
  24                                 .key_size = {
  25                                         .min = 1,
  26                                         .max = 64,
  27                                         .increment = 1
  28                                 },
  29                                 .digest_size = {
  30                                         .min = 1,
  31                                         .max = 16,
  32                                         .increment = 1
  33                                 },
  34                                 .iv_size = { 0 }
  35                         }, }
  36                 }, }
  37         },
  38         {       /* SHA1 HMAC */
  39                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  40                 {.sym = {
  41                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  42                         {.auth = {
  43                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  44                                 .block_size = 64,
  45                                 .key_size = {
  46                                         .min = 1,
  47                                         .max = 65535,
  48                                         .increment = 1
  49                                 },
  50                                 .digest_size = {
  51                                         .min = 1,
  52                                         .max = 20,
  53                                         .increment = 1
  54                                 },
  55                                 .iv_size = { 0 }
  56                         }, }
  57                 }, }
  58         },
  59         {       /* SHA1 */
  60                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  61                 {.sym = {
  62                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  63                         {.auth = {
  64                                 .algo = RTE_CRYPTO_AUTH_SHA1,
  65                                 .block_size = 64,
  66                                 .key_size = {
  67                                         .min = 0,
  68                                         .max = 0,
  69                                         .increment = 0
  70                                 },
  71                                 .digest_size = {
  72                                         .min = 1,
  73                                         .max = 20,
  74                                         .increment = 1
  75                                 },
  76                                 .iv_size = { 0 }
  77                         }, }
  78                 }, }
  79         },
  80         {       /* SHA224 HMAC */
  81                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  82                 {.sym = {
  83                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  84                         {.auth = {
  85                                 .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
  86                                 .block_size = 64,
  87                                 .key_size = {
  88                                         .min = 1,
  89                                         .max = 65535,
  90                                         .increment = 1
  91                                 },
  92                                 .digest_size = {
  93                                         .min = 1,
  94                                         .max = 28,
  95                                         .increment = 1
  96                                 },
  97                                 .iv_size = { 0 }
  98                         }, }
  99                 }, }
 100         },
 101         {       /* SHA224 */
 102                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 103                 {.sym = {
 104                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 105                         {.auth = {
 106                                 .algo = RTE_CRYPTO_AUTH_SHA224,
 107                                 .block_size = 64,
 108                                 .key_size = {
 109                                         .min = 0,
 110                                         .max = 0,
 111                                         .increment = 0
 112                                 },
 113                                 .digest_size = {
 114                                         .min = 1,
 115                                         .max = 28,
 116                                         .increment = 1
 117                                 },
 118                                 .iv_size = { 0 }
 119                         }, }
 120                 }, }
 121         },
 122         {       /* SHA256 HMAC */
 123                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 124                 {.sym = {
 125                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 126                         {.auth = {
 127                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
 128                                 .block_size = 64,
 129                                 .key_size = {
 130                                         .min = 1,
 131                                         .max = 65535,
 132                                         .increment = 1
 133                                 },
 134                                 .digest_size = {
 135                                         .min = 1,
 136                                         .max = 32,
 137                                         .increment = 1
 138                                 },
 139                                 .iv_size = { 0 }
 140                         }, }
 141                 }, }
 142         },
 143         {       /* SHA256 */
 144                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 145                 {.sym = {
 146                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 147                         {.auth = {
 148                                 .algo = RTE_CRYPTO_AUTH_SHA256,
 149                                 .block_size = 64,
 150                                 .key_size = {
 151                                         .min = 0,
 152                                         .max = 0,
 153                                         .increment = 0
 154                                 },
 155                                 .digest_size = {
 156                                         .min = 1,
 157                                         .max = 32,
 158                                         .increment = 1
 159                                 },
 160                                 .iv_size = { 0 }
 161                         }, }
 162                 }, }
 163         },
 164         {       /* SHA384 HMAC */
 165                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 166                 {.sym = {
 167                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 168                         {.auth = {
 169                                 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
 170                                 .block_size = 128,
 171                                 .key_size = {
 172                                         .min = 1,
 173                                         .max = 65535,
 174                                         .increment = 1
 175                                 },
 176                                 .digest_size = {
 177                                         .min = 1,
 178                                         .max = 48,
 179                                         .increment = 1
 180                                 },
 181                                 .iv_size = { 0 }
 182                         }, }
 183                 }, }
 184         },
 185         {       /* SHA384 */
 186                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 187                 {.sym = {
 188                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 189                         {.auth = {
 190                                 .algo = RTE_CRYPTO_AUTH_SHA384,
 191                                 .block_size = 128,
 192                                 .key_size = {
 193                                         .min = 0,
 194                                         .max = 0,
 195                                         .increment = 0
 196                                 },
 197                                 .digest_size = {
 198                                         .min = 1,
 199                                         .max = 48,
 200                                         .increment = 1
 201                                 },
 202                                 .iv_size = { 0 }
 203                         }, }
 204                 }, }
 205         },
 206         {       /* SHA512 HMAC */
 207                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 208                 {.sym = {
 209                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 210                         {.auth = {
 211                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 212                                 .block_size = 128,
 213                                 .key_size = {
 214                                         .min = 1,
 215                                         .max = 65535,
 216                                         .increment = 1
 217                                 },
 218                                 .digest_size = {
 219                                         .min = 1,
 220                                         .max = 64,
 221                                         .increment = 1
 222                                 },
 223                                 .iv_size = { 0 }
 224                         }, }
 225                 }, }
 226         },
 227         {       /* SHA512  */
 228                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 229                 {.sym = {
 230                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 231                         {.auth = {
 232                                 .algo = RTE_CRYPTO_AUTH_SHA512,
 233                                 .block_size = 128,
 234                                 .key_size = {
 235                                         .min = 0,
 236                                         .max = 0,
 237                                         .increment = 0
 238                                 },
 239                                 .digest_size = {
 240                                         .min = 1,
 241                                         .max = 64,
 242                                         .increment = 1
 243                                 },
 244                                 .iv_size = { 0 }
 245                         }, }
 246                 }, }
 247         },
 248         {       /* AES XCBC HMAC */
 249                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 250                 {.sym = {
 251                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 252                         {.auth = {
 253                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 254                                 .block_size = 16,
 255                                 .key_size = {
 256                                         .min = 16,
 257                                         .max = 16,
 258                                         .increment = 0
 259                                 },
 260                                 .digest_size = {
 261                                         .min = 12,
 262                                         .max = 12,
 263                                         .increment = 0
 264                                 },
 265                                 .iv_size = { 0 }
 266                         }, }
 267                 }, }
 268         },
 269         {       /* AES CBC */
 270                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 271                 {.sym = {
 272                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 273                         {.cipher = {
 274                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 275                                 .block_size = 16,
 276                                 .key_size = {
 277                                         .min = 16,
 278                                         .max = 32,
 279                                         .increment = 8
 280                                 },
 281                                 .iv_size = {
 282                                         .min = 16,
 283                                         .max = 16,
 284                                         .increment = 0
 285                                 }
 286                         }, }
 287                 }, }
 288         },
 289         {       /* AES CTR */
 290                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 291                 {.sym = {
 292                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 293                         {.cipher = {
 294                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 295                                 .block_size = 16,
 296                                 .key_size = {
 297                                         .min = 16,
 298                                         .max = 32,
 299                                         .increment = 8
 300                                 },
 301                                 .iv_size = {
 302                                         .min = 12,
 303                                         .max = 16,
 304                                         .increment = 4
 305                                 }
 306                         }, }
 307                 }, }
 308         },
 309         {       /* AES DOCSIS BPI */
 310                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 311                 {.sym = {
 312                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 313                         {.cipher = {
 314                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 315                                 .block_size = 16,
 316                                 .key_size = {
 317                                         .min = 16,
 318 #if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
 319                                         .max = 32,
 320                                         .increment = 16
 321 #else
 322                                         .max = 16,
 323                                         .increment = 0
 324 #endif
 325                                 },
 326                                 .iv_size = {
 327                                         .min = 16,
 328                                         .max = 16,
 329                                         .increment = 0
 330                                 }
 331                         }, }
 332                 }, }
 333         },
 334         {       /* DES CBC */
 335                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 336                 {.sym = {
 337                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 338                         {.cipher = {
 339                                 .algo = RTE_CRYPTO_CIPHER_DES_CBC,
 340                                 .block_size = 8,
 341                                 .key_size = {
 342                                         .min = 8,
 343                                         .max = 8,
 344                                         .increment = 0
 345                                 },
 346                                 .iv_size = {
 347                                         .min = 8,
 348                                         .max = 8,
 349                                         .increment = 0
 350                                 }
 351                         }, }
 352                 }, }
 353         },
 354         {       /*  3DES CBC */
 355                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 356                 {.sym = {
 357                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 358                         {.cipher = {
 359                                 .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
 360                                 .block_size = 8,
 361                                 .key_size = {
 362                                         .min = 8,
 363                                         .max = 24,
 364                                         .increment = 8
 365                                 },
 366                                 .iv_size = {
 367                                         .min = 8,
 368                                         .max = 8,
 369                                         .increment = 0
 370                                 }
 371                         }, }
 372                 }, }
 373         },
 374         {       /* DES DOCSIS BPI */
 375                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 376                 {.sym = {
 377                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 378                         {.cipher = {
 379                                 .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
 380                                 .block_size = 8,
 381                                 .key_size = {
 382                                         .min = 8,
 383                                         .max = 8,
 384                                         .increment = 0
 385                                 },
 386                                 .iv_size = {
 387                                         .min = 8,
 388                                         .max = 8,
 389                                         .increment = 0
 390                                 }
 391                         }, }
 392                 }, }
 393         },
 394         {       /* AES CCM */
 395                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 396                 {.sym = {
 397                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 398                         {.aead = {
 399                                 .algo = RTE_CRYPTO_AEAD_AES_CCM,
 400                                 .block_size = 16,
 401                                 .key_size = {
 402                                         .min = 16,
 403 #if IMB_VERSION(0, 54, 2) <= IMB_VERSION_NUM
 404                                         .max = 32,
 405                                         .increment = 16
 406 #else
 407                                         .max = 16,
 408                                         .increment = 0
 409 #endif
 410                                 },
 411                                 .digest_size = {
 412                                         .min = 4,
 413                                         .max = 16,
 414                                         .increment = 2
 415                                 },
 416                                 .aad_size = {
 417                                         .min = 0,
 418                                         .max = 46,
 419                                         .increment = 1
 420                                 },
 421                                 .iv_size = {
 422                                         .min = 7,
 423                                         .max = 13,
 424                                         .increment = 1
 425                                 },
 426                         }, }
 427                 }, }
 428         },
 429         {       /* AES CMAC */
 430                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 431                 {.sym = {
 432                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 433                         {.auth = {
 434                                 .algo = RTE_CRYPTO_AUTH_AES_CMAC,
 435                                 .block_size = 16,
 436                                 .key_size = {
 437                                         .min = 16,
 438                                         .max = 16,
 439                                         .increment = 0
 440                                 },
 441                                 .digest_size = {
 442                                         .min = 1,
 443                                         .max = 16,
 444                                         .increment = 1
 445                                 },
 446                                 .iv_size = { 0 }
 447                         }, }
 448                 }, }
 449         },
 450         {       /* AES GCM */
 451                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 452                 {.sym = {
 453                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 454                         {.aead = {
 455                                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
 456                                 .block_size = 16,
 457                                 .key_size = {
 458                                         .min = 16,
 459                                         .max = 32,
 460                                         .increment = 8
 461                                 },
 462                                 .digest_size = {
 463                                         .min = 1,
 464                                         .max = 16,
 465                                         .increment = 1
 466                                 },
 467                                 .aad_size = {
 468                                         .min = 0,
 469                                         .max = 65535,
 470                                         .increment = 1
 471                                 },
 472                                 .iv_size = {
 473                                         .min = 12,
 474                                         .max = 12,
 475                                         .increment = 0
 476                                 }
 477                         }, }
 478                 }, }
 479         },
 480         {       /* AES GMAC (AUTH) */
 481                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 482                 {.sym = {
 483                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 484                         {.auth = {
 485                                 .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 486                                 .block_size = 16,
 487                                 .key_size = {
 488                                         .min = 16,
 489                                         .max = 32,
 490                                         .increment = 8
 491                                 },
 492                                 .digest_size = {
 493                                         .min = 1,
 494                                         .max = 16,
 495                                         .increment = 1
 496                                 },
 497                                 .iv_size = {
 498                                         .min = 12,
 499                                         .max = 12,
 500                                         .increment = 0
 501                                 }
 502                         }, }
 503                 }, }
 504         },
 505 #if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
 506         {       /* AES ECB */
 507                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 508                 {.sym = {
 509                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 510                         {.cipher = {
 511                                 .algo = RTE_CRYPTO_CIPHER_AES_ECB,
 512                                 .block_size = 16,
 513                                 .key_size = {
 514                                         .min = 16,
 515                                         .max = 32,
 516                                         .increment = 8
 517                                 },
 518                                 .iv_size = { 0 }
 519                         }, }
 520                 }, }
 521         },
 522 #endif
 523 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 524         {       /* ZUC (EIA3) */
 525                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 526                 {.sym = {
 527                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 528                         {.auth = {
 529                                 .algo = RTE_CRYPTO_AUTH_ZUC_EIA3,
 530                                 .block_size = 16,
 531                                 .key_size = {
 532                                         .min = 16,
 533                                         .max = 16,
 534                                         .increment = 0
 535                                 },
 536                                 .digest_size = {
 537                                         .min = 4,
 538                                         .max = 4,
 539                                         .increment = 0
 540                                 },
 541                                 .iv_size = {
 542                                         .min = 16,
 543                                         .max = 16,
 544                                         .increment = 0
 545                                 }
 546                         }, }
 547                 }, }
 548         },
 549         {       /* ZUC (EEA3) */
 550                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 551                 {.sym = {
 552                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 553                         {.cipher = {
 554                                 .algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,
 555                                 .block_size = 16,
 556                                 .key_size = {
 557                                         .min = 16,
 558                                         .max = 16,
 559                                         .increment = 0
 560                                 },
 561                                 .iv_size = {
 562                                         .min = 16,
 563                                         .max = 16,
 564                                         .increment = 0
 565                                 },
 566                         }, }
 567                 }, }
 568         },
 569         {       /* SNOW 3G (UIA2) */
 570                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 571                 {.sym = {
 572                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 573                         {.auth = {
 574                                 .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
 575                                 .block_size = 16,
 576                                 .key_size = {
 577                                         .min = 16,
 578                                         .max = 16,
 579                                         .increment = 0
 580                                 },
 581                                 .digest_size = {
 582                                         .min = 4,
 583                                         .max = 4,
 584                                         .increment = 0
 585                                 },
 586                                 .iv_size = {
 587                                         .min = 16,
 588                                         .max = 16,
 589                                         .increment = 0
 590                                 }
 591                         }, }
 592                 }, }
 593         },
 594         {       /* SNOW 3G (UEA2) */
 595                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 596                 {.sym = {
 597                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 598                         {.cipher = {
 599                                 .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 600                                 .block_size = 16,
 601                                 .key_size = {
 602                                         .min = 16,
 603                                         .max = 16,
 604                                         .increment = 0
 605                                 },
 606                                 .iv_size = {
 607                                         .min = 16,
 608                                         .max = 16,
 609                                         .increment = 0
 610                                 }
 611                         }, }
 612                 }, }
 613         },
 614         {       /* KASUMI (F9) */
 615                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 616                 {.sym = {
 617                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 618                         {.auth = {
 619                                 .algo = RTE_CRYPTO_AUTH_KASUMI_F9,
 620                                 .block_size = 8,
 621                                 .key_size = {
 622                                         .min = 16,
 623                                         .max = 16,
 624                                         .increment = 0
 625                                 },
 626                                 .digest_size = {
 627                                         .min = 4,
 628                                         .max = 4,
 629                                         .increment = 0
 630                                 },
 631                                 .iv_size = { 0 }
 632                         }, }
 633                 }, }
 634         },
 635         {       /* KASUMI (F8) */
 636                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 637                 {.sym = {
 638                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 639                         {.cipher = {
 640                                 .algo = RTE_CRYPTO_CIPHER_KASUMI_F8,
 641                                 .block_size = 8,
 642                                 .key_size = {
 643                                         .min = 16,
 644                                         .max = 16,
 645                                         .increment = 0
 646                                 },
 647                                 .iv_size = {
 648                                         .min = 8,
 649                                         .max = 8,
 650                                         .increment = 0
 651                                 }
 652                         }, }
 653                 }, }
 654         },
 655 #endif
 656 #if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
 657         {       /* CHACHA20-POLY1305 */
 658                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 659                 {.sym = {
 660                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 661                         {.aead = {
 662                                 .algo = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,
 663                                 .block_size = 64,
 664                                 .key_size = {
 665                                         .min = 32,
 666                                         .max = 32,
 667                                         .increment = 0
 668                                 },
 669                                 .digest_size = {
 670                                         .min = 16,
 671                                         .max = 16,
 672                                         .increment = 0
 673                                 },
 674                                 .aad_size = {
 675                                         .min = 0,
 676                                         .max = 240,
 677                                         .increment = 1
 678                                 },
 679                                 .iv_size = {
 680                                         .min = 12,
 681                                         .max = 12,
 682                                         .increment = 0
 683                                 },
 684                         }, }
 685                 }, }
 686         },
 687 #endif
 688         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 689 };
 690
 691 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 692 static const struct rte_cryptodev_capabilities
 693                                         aesni_mb_pmd_security_crypto_cap[] = {
 694         {       /* AES DOCSIS BPI */
 695                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 696                 {.sym = {
 697                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 698                         {.cipher = {
 699                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
 700                                 .block_size = 16,
 701                                 .key_size = {
 702                                         .min = 16,
 703                                         .max = 32,
 704                                         .increment = 16
 705                                 },
 706                                 .iv_size = {
 707                                         .min = 16,
 708                                         .max = 16,
 709                                         .increment = 0
 710                                 }
 711                         }, }
 712                 }, }
 713         },
 714
 715         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 716 };
 717
 718 static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
 719         {       /* DOCSIS Uplink */
 720                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 721                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 722                 .docsis = {
 723                         .direction = RTE_SECURITY_DOCSIS_UPLINK
 724                 },
 725                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 726         },
 727         {       /* DOCSIS Downlink */
 728                 .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
 729                 .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
 730                 .docsis = {
 731                         .direction = RTE_SECURITY_DOCSIS_DOWNLINK
 732                 },
 733                 .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
 734         },
 735         {
 736                 .action = RTE_SECURITY_ACTION_TYPE_NONE
 737         }
 738 };
 739 #endif
 740
 741 /** Configure device */
 742 static int
 743 aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
 744                 __rte_unused struct rte_cryptodev_config *config)
 745 {
 746         return 0;
 747 }
 748
 749 /** Start device */
 750 static int
 751 aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
 752 {
 753         return 0;
 754 }
 755
 756 /** Stop device */
 757 static void
 758 aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
 759 {
 760 }
 761
 762 /** Close device */
 763 static int
 764 aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
 765 {
 766         return 0;
 767 }
 768
 769
 770 /** Get device statistics */
 771 static void
 772 aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
 773                 struct rte_cryptodev_stats *stats)
 774 {
 775         int qp_id;
 776
 777         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 778                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 779
 780                 stats->enqueued_count += qp->stats.enqueued_count;
 781                 stats->dequeued_count += qp->stats.dequeued_count;
 782
 783                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
 784                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
 785         }
 786 }
 787
 788 /** Reset device statistics */
 789 static void
 790 aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
 791 {
 792         int qp_id;
 793
 794         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
 795                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 796
 797                 memset(&qp->stats, 0, sizeof(qp->stats));
 798         }
 799 }
 800
 801
 802 /** Get device info */
 803 static void
 804 aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
 805                 struct rte_cryptodev_info *dev_info)
 806 {
 807         struct aesni_mb_private *internals = dev->data->dev_private;
 808
 809         if (dev_info != NULL) {
 810                 dev_info->driver_id = dev->driver_id;
 811                 dev_info->feature_flags = dev->feature_flags;
 812                 dev_info->capabilities = aesni_mb_pmd_capabilities;
 813                 dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
 814                 /* No limit of number of sessions */
 815                 dev_info->sym.max_nb_sessions = 0;
 816         }
 817 }
 818
 819 /** Release queue pair */
 820 static int
 821 aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 822 {
 823         struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
 824         struct rte_ring *r = NULL;
 825
 826         if (qp != NULL) {
 827                 r = rte_ring_lookup(qp->name);
 828                 if (r)
 829                         rte_ring_free(r);
 830                 if (qp->mb_mgr)
 831                         free_mb_mgr(qp->mb_mgr);
 832                 rte_free(qp);
 833                 dev->data->queue_pairs[qp_id] = NULL;
 834         }
 835         return 0;
 836 }
 837
 838 /** set a unique name for the queue pair based on it's name, dev_id and qp_id */
 839 static int
 840 aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
 841                 struct aesni_mb_qp *qp)
 842 {
 843         unsigned n = snprintf(qp->name, sizeof(qp->name),
 844                         "aesni_mb_pmd_%u_qp_%u",
 845                         dev->data->dev_id, qp->id);
 846
 847         if (n >= sizeof(qp->name))
 848                 return -1;
 849
 850         return 0;
 851 }
 852
 853 /** Create a ring to place processed operations on */
 854 static struct rte_ring *
 855 aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
 856                 unsigned int ring_size, int socket_id)
 857 {
 858         struct rte_ring *r;
 859         char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 860
 861         unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
 862
 863         if (n >= sizeof(ring_name))
 864                 return NULL;
 865
 866         r = rte_ring_lookup(ring_name);
 867         if (r) {
 868                 if (rte_ring_get_size(r) >= ring_size) {
 869                         AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
 870                         ring_name);
 871                         return r;
 872                 }
 873
 874                 AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
 875                         ring_name);
 876                 return NULL;
 877         }
 878
 879         return rte_ring_create(ring_name, ring_size, socket_id,
 880                         RING_F_SP_ENQ | RING_F_SC_DEQ);
 881 }
 882
 883 /** Setup a queue pair */
 884 static int
 885 aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 886                 const struct rte_cryptodev_qp_conf *qp_conf,
 887                 int socket_id)
 888 {
 889         struct aesni_mb_qp *qp = NULL;
 890         struct aesni_mb_private *internals = dev->data->dev_private;
 891         int ret = -1;
 892
 893         /* Free memory prior to re-allocation if needed. */
 894         if (dev->data->queue_pairs[qp_id] != NULL)
 895                 aesni_mb_pmd_qp_release(dev, qp_id);
 896
 897         /* Allocate the queue pair data structure. */
 898         qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
 899                                         RTE_CACHE_LINE_SIZE, socket_id);
 900         if (qp == NULL)
 901                 return -ENOMEM;
 902
 903         qp->id = qp_id;
 904         dev->data->queue_pairs[qp_id] = qp;
 905
 906         if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
 907                 goto qp_setup_cleanup;
 908
 909
 910         qp->mb_mgr = alloc_mb_mgr(0);
 911         if (qp->mb_mgr == NULL) {
 912                 ret = -ENOMEM;
 913                 goto qp_setup_cleanup;
 914         }
 915
 916         switch (internals->vector_mode) {
 917         case RTE_AESNI_MB_SSE:
 918                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
 919                 init_mb_mgr_sse(qp->mb_mgr);
 920                 break;
 921         case RTE_AESNI_MB_AVX:
 922                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
 923                 init_mb_mgr_avx(qp->mb_mgr);
 924                 break;
 925         case RTE_AESNI_MB_AVX2:
 926                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 927                 init_mb_mgr_avx2(qp->mb_mgr);
 928                 break;
 929         case RTE_AESNI_MB_AVX512:
 930                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
 931                 init_mb_mgr_avx512(qp->mb_mgr);
 932                 break;
 933         default:
 934                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
 935                                 internals->vector_mode);
 936                 goto qp_setup_cleanup;
 937         }
 938
 939         qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
 940                         qp_conf->nb_descriptors, socket_id);
 941         if (qp->ingress_queue == NULL) {
 942                 ret = -1;
 943                 goto qp_setup_cleanup;
 944         }
 945
 946         qp->sess_mp = qp_conf->mp_session;
 947         qp->sess_mp_priv = qp_conf->mp_session_private;
 948
 949         memset(&qp->stats, 0, sizeof(qp->stats));
 950
 951         char mp_name[RTE_MEMPOOL_NAMESIZE];
 952
 953         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 954                                 "digest_mp_%u_%u", dev->data->dev_id, qp_id);
 955         return 0;
 956
 957 qp_setup_cleanup:
 958         if (qp) {
 959                 if (qp->mb_mgr)
 960                         free_mb_mgr(qp->mb_mgr);
 961                 rte_free(qp);
 962         }
 963
 964         return ret;
 965 }
 966
 967 /** Returns the size of the aesni multi-buffer session structure */
 968 static unsigned
 969 aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 970 {
 971         return sizeof(struct aesni_mb_session);
 972 }
 973
 974 /** Configure a aesni multi-buffer session from a crypto xform chain */
 975 static int
 976 aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
 977                 struct rte_crypto_sym_xform *xform,
 978                 struct rte_cryptodev_sym_session *sess,
 979                 struct rte_mempool *mempool)
 980 {
 981         void *sess_private_data;
 982         struct aesni_mb_private *internals = dev->data->dev_private;
 983         int ret;
 984
 985         if (unlikely(sess == NULL)) {
 986                 AESNI_MB_LOG(ERR, "invalid session struct");
 987                 return -EINVAL;
 988         }
 989
 990         if (rte_mempool_get(mempool, &sess_private_data)) {
 991                 AESNI_MB_LOG(ERR,
 992                                 "Couldn't get object from session mempool");
 993                 return -ENOMEM;
 994         }
 995
 996         ret = aesni_mb_set_session_parameters(internals->mb_mgr,
 997                         sess_private_data, xform);
 998         if (ret != 0) {
 999                 AESNI_MB_LOG(ERR, "failed configure session parameters");
1000
1001                 /* Return session to mempool */
1002                 rte_mempool_put(mempool, sess_private_data);
1003                 return ret;
1004         }
1005
1006         set_sym_session_private_data(sess, dev->driver_id,
1007                         sess_private_data);
1008
1009         return 0;
1010 }
1011
1012 /** Clear the memory of session so it doesn't leave key material behind */
1013 static void
1014 aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
1015                 struct rte_cryptodev_sym_session *sess)
1016 {
1017         uint8_t index = dev->driver_id;
1018         void *sess_priv = get_sym_session_private_data(sess, index);
1019
1020         /* Zero out the whole structure */
1021         if (sess_priv) {
1022                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
1023                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1024                 set_sym_session_private_data(sess, index, NULL);
1025                 rte_mempool_put(sess_mp, sess_priv);
1026         }
1027 }
1028
1029 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
1030                 .dev_configure          = aesni_mb_pmd_config,
1031                 .dev_start              = aesni_mb_pmd_start,
1032                 .dev_stop               = aesni_mb_pmd_stop,
1033                 .dev_close              = aesni_mb_pmd_close,
1034
1035                 .stats_get              = aesni_mb_pmd_stats_get,
1036                 .stats_reset            = aesni_mb_pmd_stats_reset,
1037
1038                 .dev_infos_get          = aesni_mb_pmd_info_get,
1039
1040                 .queue_pair_setup       = aesni_mb_pmd_qp_setup,
1041                 .queue_pair_release     = aesni_mb_pmd_qp_release,
1042
1043                 .sym_cpu_process        = aesni_mb_cpu_crypto_process_bulk,
1044
1045                 .sym_session_get_size   = aesni_mb_pmd_sym_session_get_size,
1046                 .sym_session_configure  = aesni_mb_pmd_sym_session_configure,
1047                 .sym_session_clear      = aesni_mb_pmd_sym_session_clear
1048 };
1049
1050 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
1051
1052 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1053 /**
1054  * Configure a aesni multi-buffer session from a security session
1055  * configuration
1056  */
1057 static int
1058 aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
1059                 struct rte_security_session *sess,
1060                 struct rte_mempool *mempool)
1061 {
1062         void *sess_private_data;
1063         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1064         int ret;
1065
1066         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
1067                         conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
1068                 AESNI_MB_LOG(ERR, "Invalid security protocol");
1069                 return -EINVAL;
1070         }
1071
1072         if (rte_mempool_get(mempool, &sess_private_data)) {
1073                 AESNI_MB_LOG(ERR, "Couldn't get object from session mempool");
1074                 return -ENOMEM;
1075         }
1076
1077         ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
1078                         sess_private_data);
1079
1080         if (ret != 0) {
1081                 AESNI_MB_LOG(ERR, "Failed to configure session parameters");
1082
1083                 /* Return session to mempool */
1084                 rte_mempool_put(mempool, sess_private_data);
1085                 return ret;
1086         }
1087
1088         set_sec_session_private_data(sess, sess_private_data);
1089
1090         return ret;
1091 }
1092
1093 /** Clear the memory of session so it doesn't leave key material behind */
1094 static int
1095 aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
1096                 struct rte_security_session *sess)
1097 {
1098         void *sess_priv = get_sec_session_private_data(sess);
1099
1100         if (sess_priv) {
1101                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1102                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
1103                 set_sec_session_private_data(sess, NULL);
1104                 rte_mempool_put(sess_mp, sess_priv);
1105         }
1106         return 0;
1107 }
1108
1109 /** Get security capabilities for aesni multi-buffer */
1110 static const struct rte_security_capability *
1111 aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
1112 {
1113         return aesni_mb_pmd_security_cap;
1114 }
1115
1116 static struct rte_security_ops aesni_mb_pmd_sec_ops = {
1117                 .session_create = aesni_mb_pmd_sec_sess_create,
1118                 .session_update = NULL,
1119                 .session_stats_get = NULL,
1120                 .session_destroy = aesni_mb_pmd_sec_sess_destroy,
1121                 .set_pkt_metadata = NULL,
1122                 .capabilities_get = aesni_mb_pmd_sec_capa_get
1123 };
1124
1125 struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
1126 #endif