drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <intel-ipsec-mb.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_hexdump.h>
   9 #include <rte_cryptodev.h>
  10 #include <rte_cryptodev_pmd.h>
  11 #include <rte_bus_vdev.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cpuflags.h>
  14 #include <rte_per_lcore.h>
  15 #include <rte_ether.h>
  16
  17 #include "aesni_mb_pmd_private.h"
  18
  19 #define AES_CCM_DIGEST_MIN_LEN 4
  20 #define AES_CCM_DIGEST_MAX_LEN 16
  21 #define HMAC_MAX_BLOCK_SIZE 128
  22 static uint8_t cryptodev_driver_id;
  23
  24 /*
  25  * Needed to support CPU-CRYPTO API (rte_cryptodev_sym_cpu_crypto_process),
  26  * as we still use JOB based API even for synchronous processing.
  27  */
  28 static RTE_DEFINE_PER_LCORE(MB_MGR *, sync_mb_mgr);
  29
  30 typedef void (*hash_one_block_t)(const void *data, void *digest);
  31 typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
  32
  33 /**
  34  * Calculate the authentication pre-computes
  35  *
  36  * @param one_block_hash        Function pointer to calculate digest on ipad/opad
  37  * @param ipad                  Inner pad output byte array
  38  * @param opad                  Outer pad output byte array
  39  * @param hkey                  Authentication key
  40  * @param hkey_len              Authentication key length
  41  * @param blocksize             Block size of selected hash algo
  42  */
  43 static void
  44 calculate_auth_precomputes(hash_one_block_t one_block_hash,
  45                 uint8_t *ipad, uint8_t *opad,
  46                 const uint8_t *hkey, uint16_t hkey_len,
  47                 uint16_t blocksize)
  48 {
  49         unsigned i, length;
  50
  51         uint8_t ipad_buf[blocksize] __rte_aligned(16);
  52         uint8_t opad_buf[blocksize] __rte_aligned(16);
  53
  54         /* Setup inner and outer pads */
  55         memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
  56         memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
  57
  58         /* XOR hash key with inner and outer pads */
  59         length = hkey_len > blocksize ? blocksize : hkey_len;
  60
  61         for (i = 0; i < length; i++) {
  62                 ipad_buf[i] ^= hkey[i];
  63                 opad_buf[i] ^= hkey[i];
  64         }
  65
  66         /* Compute partial hashes */
  67         (*one_block_hash)(ipad_buf, ipad);
  68         (*one_block_hash)(opad_buf, opad);
  69
  70         /* Clean up stack */
  71         memset(ipad_buf, 0, blocksize);
  72         memset(opad_buf, 0, blocksize);
  73 }
  74
  75 /** Get xform chain order */
  76 static enum aesni_mb_operation
  77 aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
  78 {
  79         if (xform == NULL)
  80                 return AESNI_MB_OP_NOT_SUPPORTED;
  81
  82         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  83                 if (xform->next == NULL)
  84                         return AESNI_MB_OP_CIPHER_ONLY;
  85                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  86                         return AESNI_MB_OP_CIPHER_HASH;
  87         }
  88
  89         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  90                 if (xform->next == NULL)
  91                         return AESNI_MB_OP_HASH_ONLY;
  92                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
  93                         return AESNI_MB_OP_HASH_CIPHER;
  94         }
  95 #if IMB_VERSION_NUM > IMB_VERSION(0, 52, 0)
  96         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  97                 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
  98                         /*
  99                          * CCM requires to hash first and cipher later
 100                          * when encrypting
 101                          */
 102                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 103                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 104                         else
 105                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 106                 } else {
 107                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 108                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 109                         else
 110                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 111                 }
 112         }
 113 #else
 114         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 115                 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
 116                                 xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
 117                         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 118                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 119                         else
 120                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 121                 }
 122         }
 123 #endif
 124
 125         return AESNI_MB_OP_NOT_SUPPORTED;
 126 }
 127
 128 /** Set session authentication parameters */
 129 static int
 130 aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
 131                 struct aesni_mb_session *sess,
 132                 const struct rte_crypto_sym_xform *xform)
 133 {
 134         hash_one_block_t hash_oneblock_fn = NULL;
 135         unsigned int key_larger_block_size = 0;
 136         uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
 137         uint32_t auth_precompute = 1;
 138
 139         if (xform == NULL) {
 140                 sess->auth.algo = NULL_HASH;
 141                 return 0;
 142         }
 143
 144         if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
 145                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
 146                 return -1;
 147         }
 148
 149         /* Set IV parameters */
 150         sess->auth_iv.offset = xform->auth.iv.offset;
 151         sess->auth_iv.length = xform->auth.iv.length;
 152
 153         /* Set the request digest size */
 154         sess->auth.req_digest_len = xform->auth.digest_length;
 155
 156         /* Select auth generate/verify */
 157         sess->auth.operation = xform->auth.op;
 158
 159         /* Set Authentication Parameters */
 160         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
 161                 sess->auth.algo = AES_XCBC;
 162
 163                 uint16_t xcbc_mac_digest_len =
 164                         get_truncated_digest_byte_length(AES_XCBC);
 165                 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
 166                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 167                         return -EINVAL;
 168                 }
 169                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 170
 171                 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
 172                                 sess->auth.xcbc.k1_expanded,
 173                                 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
 174                 return 0;
 175         }
 176
 177         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
 178                 uint32_t dust[4*15];
 179
 180                 sess->auth.algo = AES_CMAC;
 181
 182                 uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
 183
 184                 if (sess->auth.req_digest_len > cmac_digest_len) {
 185                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 186                         return -EINVAL;
 187                 }
 188                 /*
 189                  * Multi-buffer lib supports digest sizes from 4 to 16 bytes
 190                  * in version 0.50 and sizes of 12 and 16 bytes,
 191                  * in version 0.49.
 192                  * If size requested is different, generate the full digest
 193                  * (16 bytes) in a temporary location and then memcpy
 194                  * the requested number of bytes.
 195                  */
 196                 if (sess->auth.req_digest_len < 4)
 197                         sess->auth.gen_digest_len = cmac_digest_len;
 198                 else
 199                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 200
 201                 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
 202                                 sess->auth.cmac.expkey, dust);
 203                 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
 204                                 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
 205                 return 0;
 206         }
 207
 208         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 209                 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 210                         sess->cipher.direction = ENCRYPT;
 211                         sess->chain_order = CIPHER_HASH;
 212                 } else
 213                         sess->cipher.direction = DECRYPT;
 214
 215                 sess->auth.algo = AES_GMAC;
 216                 /*
 217                  * Multi-buffer lib supports 8, 12 and 16 bytes of digest.
 218                  * If size requested is different, generate the full digest
 219                  * (16 bytes) in a temporary location and then memcpy
 220                  * the requested number of bytes.
 221                  */
 222                 if (sess->auth.req_digest_len != 16 &&
 223                                 sess->auth.req_digest_len != 12 &&
 224                                 sess->auth.req_digest_len != 8) {
 225                         sess->auth.gen_digest_len = 16;
 226                 } else {
 227                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 228                 }
 229                 sess->iv.length = xform->auth.iv.length;
 230                 sess->iv.offset = xform->auth.iv.offset;
 231
 232                 switch (xform->auth.key.length) {
 233                 case AES_128_BYTES:
 234                         IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
 235                                 &sess->cipher.gcm_key);
 236                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 237                         break;
 238                 case AES_192_BYTES:
 239                         IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
 240                                 &sess->cipher.gcm_key);
 241                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 242                         break;
 243                 case AES_256_BYTES:
 244                         IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
 245                                 &sess->cipher.gcm_key);
 246                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 247                         break;
 248                 default:
 249                         RTE_LOG(ERR, PMD, "failed to parse test type\n");
 250                         return -EINVAL;
 251                 }
 252
 253                 return 0;
 254         }
 255
 256 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 257         if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 258                 sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;
 259                 uint16_t zuc_eia3_digest_len =
 260                         get_truncated_digest_byte_length(IMB_AUTH_ZUC_EIA3_BITLEN);
 261                 if (sess->auth.req_digest_len != zuc_eia3_digest_len) {
 262                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 263                         return -EINVAL;
 264                 }
 265                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 266
 267                 memcpy(sess->auth.zuc_auth_key, xform->auth.key.data, 16);
 268                 return 0;
 269         }
 270 #endif
 271
 272         switch (xform->auth.algo) {
 273         case RTE_CRYPTO_AUTH_MD5_HMAC:
 274                 sess->auth.algo = MD5;
 275                 hash_oneblock_fn = mb_mgr->md5_one_block;
 276                 break;
 277         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 278                 sess->auth.algo = SHA1;
 279                 hash_oneblock_fn = mb_mgr->sha1_one_block;
 280                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
 281                         IMB_SHA1(mb_mgr,
 282                                 xform->auth.key.data,
 283                                 xform->auth.key.length,
 284                                 hashed_key);
 285                         key_larger_block_size = 1;
 286                 }
 287                 break;
 288         case RTE_CRYPTO_AUTH_SHA1:
 289                 sess->auth.algo = PLAIN_SHA1;
 290                 auth_precompute = 0;
 291                 break;
 292         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 293                 sess->auth.algo = SHA_224;
 294                 hash_oneblock_fn = mb_mgr->sha224_one_block;
 295                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
 296                         IMB_SHA224(mb_mgr,
 297                                 xform->auth.key.data,
 298                                 xform->auth.key.length,
 299                                 hashed_key);
 300                         key_larger_block_size = 1;
 301                 }
 302                 break;
 303         case RTE_CRYPTO_AUTH_SHA224:
 304                 sess->auth.algo = PLAIN_SHA_224;
 305                 auth_precompute = 0;
 306                 break;
 307         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 308                 sess->auth.algo = SHA_256;
 309                 hash_oneblock_fn = mb_mgr->sha256_one_block;
 310                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
 311                         IMB_SHA256(mb_mgr,
 312                                 xform->auth.key.data,
 313                                 xform->auth.key.length,
 314                                 hashed_key);
 315                         key_larger_block_size = 1;
 316                 }
 317                 break;
 318         case RTE_CRYPTO_AUTH_SHA256:
 319                 sess->auth.algo = PLAIN_SHA_256;
 320                 auth_precompute = 0;
 321                 break;
 322         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 323                 sess->auth.algo = SHA_384;
 324                 hash_oneblock_fn = mb_mgr->sha384_one_block;
 325                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
 326                         IMB_SHA384(mb_mgr,
 327                                 xform->auth.key.data,
 328                                 xform->auth.key.length,
 329                                 hashed_key);
 330                         key_larger_block_size = 1;
 331                 }
 332                 break;
 333         case RTE_CRYPTO_AUTH_SHA384:
 334                 sess->auth.algo = PLAIN_SHA_384;
 335                 auth_precompute = 0;
 336                 break;
 337         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 338                 sess->auth.algo = SHA_512;
 339                 hash_oneblock_fn = mb_mgr->sha512_one_block;
 340                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
 341                         IMB_SHA512(mb_mgr,
 342                                 xform->auth.key.data,
 343                                 xform->auth.key.length,
 344                                 hashed_key);
 345                         key_larger_block_size = 1;
 346                 }
 347                 break;
 348         case RTE_CRYPTO_AUTH_SHA512:
 349                 sess->auth.algo = PLAIN_SHA_512;
 350                 auth_precompute = 0;
 351                 break;
 352         default:
 353                 AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
 354                 return -ENOTSUP;
 355         }
 356         uint16_t trunc_digest_size =
 357                         get_truncated_digest_byte_length(sess->auth.algo);
 358         uint16_t full_digest_size =
 359                         get_digest_byte_length(sess->auth.algo);
 360
 361         if (sess->auth.req_digest_len > full_digest_size ||
 362                         sess->auth.req_digest_len == 0) {
 363                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 364                 return -EINVAL;
 365         }
 366
 367         if (sess->auth.req_digest_len != trunc_digest_size &&
 368                         sess->auth.req_digest_len != full_digest_size)
 369                 sess->auth.gen_digest_len = full_digest_size;
 370         else
 371                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 372
 373         /* Plain SHA does not require precompute key */
 374         if (auth_precompute == 0)
 375                 return 0;
 376
 377         /* Calculate Authentication precomputes */
 378         if (key_larger_block_size) {
 379                 calculate_auth_precomputes(hash_oneblock_fn,
 380                         sess->auth.pads.inner, sess->auth.pads.outer,
 381                         hashed_key,
 382                         xform->auth.key.length,
 383                         get_auth_algo_blocksize(sess->auth.algo));
 384         } else {
 385                 calculate_auth_precomputes(hash_oneblock_fn,
 386                         sess->auth.pads.inner, sess->auth.pads.outer,
 387                         xform->auth.key.data,
 388                         xform->auth.key.length,
 389                         get_auth_algo_blocksize(sess->auth.algo));
 390         }
 391
 392         return 0;
 393 }
 394
 395 /** Set session cipher parameters */
 396 static int
 397 aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
 398                 struct aesni_mb_session *sess,
 399                 const struct rte_crypto_sym_xform *xform)
 400 {
 401         uint8_t is_aes = 0;
 402         uint8_t is_3DES = 0;
 403         uint8_t is_docsis = 0;
 404 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 405         uint8_t is_zuc = 0;
 406 #endif
 407
 408         if (xform == NULL) {
 409                 sess->cipher.mode = NULL_CIPHER;
 410                 return 0;
 411         }
 412
 413         if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
 414                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
 415                 return -EINVAL;
 416         }
 417
 418         /* Select cipher direction */
 419         switch (xform->cipher.op) {
 420         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
 421                 sess->cipher.direction = ENCRYPT;
 422                 break;
 423         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
 424                 sess->cipher.direction = DECRYPT;
 425                 break;
 426         default:
 427                 AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
 428                 return -EINVAL;
 429         }
 430
 431         /* Select cipher mode */
 432         switch (xform->cipher.algo) {
 433         case RTE_CRYPTO_CIPHER_AES_CBC:
 434                 sess->cipher.mode = CBC;
 435                 is_aes = 1;
 436                 break;
 437         case RTE_CRYPTO_CIPHER_AES_CTR:
 438                 sess->cipher.mode = CNTR;
 439                 is_aes = 1;
 440                 break;
 441         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 442                 sess->cipher.mode = DOCSIS_SEC_BPI;
 443                 is_docsis = 1;
 444                 break;
 445         case RTE_CRYPTO_CIPHER_DES_CBC:
 446                 sess->cipher.mode = DES;
 447                 break;
 448         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 449                 sess->cipher.mode = DOCSIS_DES;
 450                 break;
 451         case RTE_CRYPTO_CIPHER_3DES_CBC:
 452                 sess->cipher.mode = DES3;
 453                 is_3DES = 1;
 454                 break;
 455 #if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
 456         case RTE_CRYPTO_CIPHER_AES_ECB:
 457                 sess->cipher.mode = ECB;
 458                 is_aes = 1;
 459                 break;
 460 #endif
 461 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 462         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 463                 sess->cipher.mode = IMB_CIPHER_ZUC_EEA3;
 464                 is_zuc = 1;
 465                 break;
 466 #endif
 467         default:
 468                 AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
 469                 return -ENOTSUP;
 470         }
 471
 472         /* Set IV parameters */
 473         sess->iv.offset = xform->cipher.iv.offset;
 474         sess->iv.length = xform->cipher.iv.length;
 475
 476         /* Check key length and choose key expansion function for AES */
 477         if (is_aes) {
 478                 switch (xform->cipher.key.length) {
 479                 case AES_128_BYTES:
 480                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 481                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 482                                         sess->cipher.expanded_aes_keys.encode,
 483                                         sess->cipher.expanded_aes_keys.decode);
 484                         break;
 485                 case AES_192_BYTES:
 486                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 487                         IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
 488                                         sess->cipher.expanded_aes_keys.encode,
 489                                         sess->cipher.expanded_aes_keys.decode);
 490                         break;
 491                 case AES_256_BYTES:
 492                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 493                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 494                                         sess->cipher.expanded_aes_keys.encode,
 495                                         sess->cipher.expanded_aes_keys.decode);
 496                         break;
 497                 default:
 498                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 499                         return -EINVAL;
 500                 }
 501         } else if (is_docsis) {
 502                 switch (xform->cipher.key.length) {
 503                 case AES_128_BYTES:
 504                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 505                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 506                                         sess->cipher.expanded_aes_keys.encode,
 507                                         sess->cipher.expanded_aes_keys.decode);
 508                         break;
 509 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 510                 case AES_256_BYTES:
 511                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 512                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 513                                         sess->cipher.expanded_aes_keys.encode,
 514                                         sess->cipher.expanded_aes_keys.decode);
 515                         break;
 516 #endif
 517                 default:
 518                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 519                         return -EINVAL;
 520                 }
 521         } else if (is_3DES) {
 522                 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
 523                                 sess->cipher.exp_3des_keys.key[1],
 524                                 sess->cipher.exp_3des_keys.key[2]};
 525
 526                 switch (xform->cipher.key.length) {
 527                 case  24:
 528                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 529                                         xform->cipher.key.data);
 530                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 531                                         xform->cipher.key.data + 8);
 532                         IMB_DES_KEYSCHED(mb_mgr, keys[2],
 533                                         xform->cipher.key.data + 16);
 534
 535                         /* Initialize keys - 24 bytes: [K1-K2-K3] */
 536                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 537                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 538                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
 539                         break;
 540                 case 16:
 541                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 542                                         xform->cipher.key.data);
 543                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 544                                         xform->cipher.key.data + 8);
 545                         /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
 546                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 547                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 548                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 549                         break;
 550                 case 8:
 551                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 552                                         xform->cipher.key.data);
 553
 554                         /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
 555                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 556                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
 557                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 558                         break;
 559                 default:
 560                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 561                         return -EINVAL;
 562                 }
 563
 564                 sess->cipher.key_length_in_bytes = 24;
 565 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 566         } else if (is_zuc) {
 567                 if (xform->cipher.key.length != 16) {
 568                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 569                         return -EINVAL;
 570                 }
 571                 sess->cipher.key_length_in_bytes = 16;
 572                 memcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data,
 573                         16);
 574 #endif
 575         } else {
 576                 if (xform->cipher.key.length != 8) {
 577                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 578                         return -EINVAL;
 579                 }
 580                 sess->cipher.key_length_in_bytes = 8;
 581
 582                 IMB_DES_KEYSCHED(mb_mgr,
 583                         (uint64_t *)sess->cipher.expanded_aes_keys.encode,
 584                                 xform->cipher.key.data);
 585                 IMB_DES_KEYSCHED(mb_mgr,
 586                         (uint64_t *)sess->cipher.expanded_aes_keys.decode,
 587                                 xform->cipher.key.data);
 588         }
 589
 590         return 0;
 591 }
 592
 593 static int
 594 aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
 595                 struct aesni_mb_session *sess,
 596                 const struct rte_crypto_sym_xform *xform)
 597 {
 598         switch (xform->aead.op) {
 599         case RTE_CRYPTO_AEAD_OP_ENCRYPT:
 600                 sess->cipher.direction = ENCRYPT;
 601                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 602                 break;
 603         case RTE_CRYPTO_AEAD_OP_DECRYPT:
 604                 sess->cipher.direction = DECRYPT;
 605                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 606                 break;
 607         default:
 608                 AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
 609                 return -EINVAL;
 610         }
 611
 612         switch (xform->aead.algo) {
 613         case RTE_CRYPTO_AEAD_AES_CCM:
 614                 sess->cipher.mode = CCM;
 615                 sess->auth.algo = AES_CCM;
 616
 617                 /* Check key length and choose key expansion function for AES */
 618                 switch (xform->aead.key.length) {
 619                 case AES_128_BYTES:
 620                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 621                         IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
 622                                         sess->cipher.expanded_aes_keys.encode,
 623                                         sess->cipher.expanded_aes_keys.decode);
 624                         break;
 625                 default:
 626                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 627                         return -EINVAL;
 628                 }
 629
 630                 break;
 631
 632         case RTE_CRYPTO_AEAD_AES_GCM:
 633                 sess->cipher.mode = GCM;
 634                 sess->auth.algo = AES_GMAC;
 635
 636                 switch (xform->aead.key.length) {
 637                 case AES_128_BYTES:
 638                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 639                         IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
 640                                 &sess->cipher.gcm_key);
 641                         break;
 642                 case AES_192_BYTES:
 643                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 644                         IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
 645                                 &sess->cipher.gcm_key);
 646                         break;
 647                 case AES_256_BYTES:
 648                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 649                         IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
 650                                 &sess->cipher.gcm_key);
 651                         break;
 652                 default:
 653                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 654                         return -EINVAL;
 655                 }
 656
 657                 break;
 658
 659         default:
 660                 AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
 661                 return -ENOTSUP;
 662         }
 663
 664         /* Set IV parameters */
 665         sess->iv.offset = xform->aead.iv.offset;
 666         sess->iv.length = xform->aead.iv.length;
 667
 668         sess->auth.req_digest_len = xform->aead.digest_length;
 669         /* CCM digests must be between 4 and 16 and an even number */
 670         if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
 671                         sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
 672                         (sess->auth.req_digest_len & 1) == 1) {
 673                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 674                 return -EINVAL;
 675         }
 676         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 677
 678         return 0;
 679 }
 680
 681 /** Parse crypto xform chain and set private session parameters */
 682 int
 683 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 684                 struct aesni_mb_session *sess,
 685                 const struct rte_crypto_sym_xform *xform)
 686 {
 687         const struct rte_crypto_sym_xform *auth_xform = NULL;
 688         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 689         const struct rte_crypto_sym_xform *aead_xform = NULL;
 690         int ret;
 691
 692         /* Select Crypto operation - hash then cipher / cipher then hash */
 693         switch (aesni_mb_get_chain_order(xform)) {
 694         case AESNI_MB_OP_HASH_CIPHER:
 695                 sess->chain_order = HASH_CIPHER;
 696                 auth_xform = xform;
 697                 cipher_xform = xform->next;
 698                 break;
 699         case AESNI_MB_OP_CIPHER_HASH:
 700                 sess->chain_order = CIPHER_HASH;
 701                 auth_xform = xform->next;
 702                 cipher_xform = xform;
 703                 break;
 704         case AESNI_MB_OP_HASH_ONLY:
 705                 sess->chain_order = HASH_CIPHER;
 706                 auth_xform = xform;
 707                 cipher_xform = NULL;
 708                 break;
 709         case AESNI_MB_OP_CIPHER_ONLY:
 710                 /*
 711                  * Multi buffer library operates only at two modes,
 712                  * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
 713                  * chain order depends on cipher operation: encryption is always
 714                  * the first operation and decryption the last one.
 715                  */
 716                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 717                         sess->chain_order = CIPHER_HASH;
 718                 else
 719                         sess->chain_order = HASH_CIPHER;
 720                 auth_xform = NULL;
 721                 cipher_xform = xform;
 722                 break;
 723         case AESNI_MB_OP_AEAD_CIPHER_HASH:
 724                 sess->chain_order = CIPHER_HASH;
 725                 sess->aead.aad_len = xform->aead.aad_length;
 726                 aead_xform = xform;
 727                 break;
 728         case AESNI_MB_OP_AEAD_HASH_CIPHER:
 729                 sess->chain_order = HASH_CIPHER;
 730                 sess->aead.aad_len = xform->aead.aad_length;
 731                 aead_xform = xform;
 732                 break;
 733         case AESNI_MB_OP_NOT_SUPPORTED:
 734         default:
 735                 AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
 736                 return -ENOTSUP;
 737         }
 738
 739         /* Default IV length = 0 */
 740         sess->iv.length = 0;
 741         sess->auth_iv.length = 0;
 742
 743         ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
 744         if (ret != 0) {
 745                 AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
 746                 return ret;
 747         }
 748
 749         ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
 750                         cipher_xform);
 751         if (ret != 0) {
 752                 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 753                 return ret;
 754         }
 755
 756         if (aead_xform) {
 757                 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
 758                                 aead_xform);
 759                 if (ret != 0) {
 760                         AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
 761                         return ret;
 762                 }
 763         }
 764
 765         return 0;
 766 }
 767
 768 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 769 /** Check DOCSIS security session configuration is valid */
 770 static int
 771 check_docsis_sec_session(struct rte_security_session_conf *conf)
 772 {
 773         struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;
 774         struct rte_security_docsis_xform *docsis = &conf->docsis;
 775
 776         /* Downlink: CRC generate -> Cipher encrypt */
 777         if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
 778
 779                 if (crypto_sym != NULL &&
 780                     crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 781                     crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
 782                     crypto_sym->cipher.algo ==
 783                                         RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
 784                     (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||
 785                      crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&
 786                     crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
 787                     crypto_sym->next == NULL) {
 788                         return 0;
 789                 }
 790         /* Uplink: Cipher decrypt -> CRC verify */
 791         } else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {
 792
 793                 if (crypto_sym != NULL &&
 794                     crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 795                     crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
 796                     crypto_sym->cipher.algo ==
 797                                         RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
 798                     (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||
 799                      crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&
 800                     crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
 801                     crypto_sym->next == NULL) {
 802                         return 0;
 803                 }
 804         }
 805
 806         return -EINVAL;
 807 }
 808
 809 /** Set DOCSIS security session auth (CRC) parameters */
 810 static int
 811 aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
 812                 struct rte_security_docsis_xform *xform)
 813 {
 814         if (xform == NULL) {
 815                 AESNI_MB_LOG(ERR, "Invalid DOCSIS xform");
 816                 return -EINVAL;
 817         }
 818
 819         /* Select CRC generate/verify */
 820         if (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) {
 821                 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
 822                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 823         } else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
 824                 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
 825                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 826         } else {
 827                 AESNI_MB_LOG(ERR, "Unsupported DOCSIS direction");
 828                 return -ENOTSUP;
 829         }
 830
 831         sess->auth.req_digest_len = RTE_ETHER_CRC_LEN;
 832         sess->auth.gen_digest_len = RTE_ETHER_CRC_LEN;
 833
 834         return 0;
 835 }
 836
 837 /**
 838  * Parse DOCSIS security session configuration and set private session
 839  * parameters
 840  */
 841 int
 842 aesni_mb_set_docsis_sec_session_parameters(
 843                 __rte_unused struct rte_cryptodev *dev,
 844                 struct rte_security_session_conf *conf,
 845                 void *sess)
 846 {
 847         struct rte_security_docsis_xform *docsis_xform;
 848         struct rte_crypto_sym_xform *cipher_xform;
 849         struct aesni_mb_session *aesni_sess = sess;
 850         struct aesni_mb_private *internals = dev->data->dev_private;
 851         int ret;
 852
 853         ret = check_docsis_sec_session(conf);
 854         if (ret) {
 855                 AESNI_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
 856                 return ret;
 857         }
 858
 859         switch (conf->docsis.direction) {
 860         case RTE_SECURITY_DOCSIS_UPLINK:
 861                 aesni_sess->chain_order = IMB_ORDER_CIPHER_HASH;
 862                 docsis_xform = &conf->docsis;
 863                 cipher_xform = conf->crypto_xform;
 864                 break;
 865         case RTE_SECURITY_DOCSIS_DOWNLINK:
 866                 aesni_sess->chain_order = IMB_ORDER_HASH_CIPHER;
 867                 cipher_xform = conf->crypto_xform;
 868                 docsis_xform = &conf->docsis;
 869                 break;
 870         default:
 871                 return -EINVAL;
 872         }
 873
 874         /* Default IV length = 0 */
 875         aesni_sess->iv.length = 0;
 876
 877         ret = aesni_mb_set_docsis_sec_session_auth_parameters(aesni_sess,
 878                         docsis_xform);
 879         if (ret != 0) {
 880                 AESNI_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
 881                 return -EINVAL;
 882         }
 883
 884         ret = aesni_mb_set_session_cipher_parameters(internals->mb_mgr,
 885                         aesni_sess, cipher_xform);
 886
 887         if (ret != 0) {
 888                 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 889                 return -EINVAL;
 890         }
 891
 892         return 0;
 893 }
 894 #endif
 895
 896 /**
 897  * burst enqueue, place crypto operations on ingress queue for processing.
 898  *
 899  * @param __qp         Queue Pair to process
 900  * @param ops          Crypto operations for processing
 901  * @param nb_ops       Number of crypto operations for processing
 902  *
 903  * @return
 904  * - Number of crypto operations enqueued
 905  */
 906 static uint16_t
 907 aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
 908                 uint16_t nb_ops)
 909 {
 910         struct aesni_mb_qp *qp = __qp;
 911
 912         unsigned int nb_enqueued;
 913
 914         nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
 915                         (void **)ops, nb_ops, NULL);
 916
 917         qp->stats.enqueued_count += nb_enqueued;
 918
 919         return nb_enqueued;
 920 }
 921
 922 /** Get multi buffer session */
 923 static inline struct aesni_mb_session *
 924 get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
 925 {
 926         struct aesni_mb_session *sess = NULL;
 927
 928         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 929                 if (likely(op->sym->session != NULL))
 930                         sess = (struct aesni_mb_session *)
 931                                         get_sym_session_private_data(
 932                                         op->sym->session,
 933                                         cryptodev_driver_id);
 934 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 935         } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 936                 if (likely(op->sym->sec_session != NULL))
 937                         sess = (struct aesni_mb_session *)
 938                                         get_sec_session_private_data(
 939                                                 op->sym->sec_session);
 940 #endif
 941         } else {
 942                 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
 943                 void *_sess_private_data = NULL;
 944
 945                 if (_sess == NULL)
 946                         return NULL;
 947
 948                 if (rte_mempool_get(qp->sess_mp_priv,
 949                                 (void **)&_sess_private_data))
 950                         return NULL;
 951
 952                 sess = (struct aesni_mb_session *)_sess_private_data;
 953
 954                 if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
 955                                 sess, op->sym->xform) != 0)) {
 956                         rte_mempool_put(qp->sess_mp, _sess);
 957                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 958                         sess = NULL;
 959                 }
 960                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 961                 set_sym_session_private_data(op->sym->session,
 962                                 cryptodev_driver_id, _sess_private_data);
 963         }
 964
 965         if (unlikely(sess == NULL))
 966                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 967
 968         return sess;
 969 }
 970
 971 static inline uint64_t
 972 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 973                 uint32_t oop)
 974 {
 975         struct rte_mbuf *m_src, *m_dst;
 976         uint8_t *p_src, *p_dst;
 977         uintptr_t u_src, u_dst;
 978         uint32_t cipher_end, auth_end;
 979
 980         /* Only cipher then hash needs special calculation. */
 981         if (!oop || session->chain_order != CIPHER_HASH)
 982                 return op->sym->auth.data.offset;
 983
 984         m_src = op->sym->m_src;
 985         m_dst = op->sym->m_dst;
 986
 987         p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
 988         p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
 989         u_src = (uintptr_t)p_src;
 990         u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
 991
 992         /**
 993          * Copy the content between cipher offset and auth offset for generating
 994          * correct digest.
 995          */
 996         if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
 997                 memcpy(p_dst + op->sym->auth.data.offset,
 998                                 p_src + op->sym->auth.data.offset,
 999                                 op->sym->cipher.data.offset -
1000                                 op->sym->auth.data.offset);
1001
1002         /**
1003          * Copy the content between (cipher offset + length) and (auth offset +
1004          * length) for generating correct digest
1005          */
1006         cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
1007         auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
1008         if (cipher_end < auth_end)
1009                 memcpy(p_dst + cipher_end, p_src + cipher_end,
1010                                 auth_end - cipher_end);
1011
1012         /**
1013          * Since intel-ipsec-mb only supports positive values,
1014          * we need to deduct the correct offset between src and dst.
1015          */
1016
1017         return u_src < u_dst ? (u_dst - u_src) :
1018                         (UINT64_MAX - u_src + u_dst + 1);
1019 }
1020
1021 static inline void
1022 set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
1023                 union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
1024                 void *iv, void *aad, void *digest, void *udata)
1025 {
1026         /* Set crypto operation */
1027         job->chain_order = session->chain_order;
1028
1029         /* Set cipher parameters */
1030         job->cipher_direction = session->cipher.direction;
1031         job->cipher_mode = session->cipher.mode;
1032
1033         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
1034
1035         /* Set authentication parameters */
1036         job->hash_alg = session->auth.algo;
1037         job->iv = iv;
1038
1039         switch (job->hash_alg) {
1040         case AES_XCBC:
1041                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
1042                 job->u.XCBC._k2 = session->auth.xcbc.k2;
1043                 job->u.XCBC._k3 = session->auth.xcbc.k3;
1044
1045                 job->aes_enc_key_expanded =
1046                                 session->cipher.expanded_aes_keys.encode;
1047                 job->aes_dec_key_expanded =
1048                                 session->cipher.expanded_aes_keys.decode;
1049                 break;
1050
1051         case AES_CCM:
1052                 job->u.CCM.aad = (uint8_t *)aad + 18;
1053                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1054                 job->aes_enc_key_expanded =
1055                                 session->cipher.expanded_aes_keys.encode;
1056                 job->aes_dec_key_expanded =
1057                                 session->cipher.expanded_aes_keys.decode;
1058                 job->iv++;
1059                 break;
1060
1061         case AES_CMAC:
1062                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1063                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1064                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1065                 job->aes_enc_key_expanded =
1066                                 session->cipher.expanded_aes_keys.encode;
1067                 job->aes_dec_key_expanded =
1068                                 session->cipher.expanded_aes_keys.decode;
1069                 break;
1070
1071         case AES_GMAC:
1072                 if (session->cipher.mode == GCM) {
1073                         job->u.GCM.aad = aad;
1074                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1075                 } else {
1076                         /* For GMAC */
1077                         job->u.GCM.aad = buf;
1078                         job->u.GCM.aad_len_in_bytes = len;
1079                         job->cipher_mode = GCM;
1080                 }
1081                 job->aes_enc_key_expanded = &session->cipher.gcm_key;
1082                 job->aes_dec_key_expanded = &session->cipher.gcm_key;
1083                 break;
1084
1085         default:
1086                 job->u.HMAC._hashed_auth_key_xor_ipad =
1087                                 session->auth.pads.inner;
1088                 job->u.HMAC._hashed_auth_key_xor_opad =
1089                                 session->auth.pads.outer;
1090
1091                 if (job->cipher_mode == DES3) {
1092                         job->aes_enc_key_expanded =
1093                                 session->cipher.exp_3des_keys.ks_ptr;
1094                         job->aes_dec_key_expanded =
1095                                 session->cipher.exp_3des_keys.ks_ptr;
1096                 } else {
1097                         job->aes_enc_key_expanded =
1098                                 session->cipher.expanded_aes_keys.encode;
1099                         job->aes_dec_key_expanded =
1100                                 session->cipher.expanded_aes_keys.decode;
1101                 }
1102         }
1103
1104         /*
1105          * Multi-buffer library current only support returning a truncated
1106          * digest length as specified in the relevant IPsec RFCs
1107          */
1108
1109         /* Set digest location and length */
1110         job->auth_tag_output = digest;
1111         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1112
1113         /* Set IV parameters */
1114         job->iv_len_in_bytes = session->iv.length;
1115
1116         /* Data Parameters */
1117         job->src = buf;
1118         job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;
1119         job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;
1120         job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;
1121         if (job->hash_alg == AES_GMAC && session->cipher.mode != GCM) {
1122                 job->msg_len_to_hash_in_bytes = 0;
1123                 job->msg_len_to_cipher_in_bytes = 0;
1124         } else {
1125                 job->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -
1126                         sofs.ofs.auth.tail;
1127                 job->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -
1128                         sofs.ofs.cipher.tail;
1129         }
1130
1131         job->user_data = udata;
1132 }
1133
1134 /**
1135  * Process a crypto operation and complete a JOB_AES_HMAC job structure for
1136  * submission to the multi buffer library for processing.
1137  *
1138  * @param       qp      queue pair
1139  * @param       job     JOB_AES_HMAC structure to fill
1140  * @param       m       mbuf to process
1141  *
1142  * @return
1143  * - Completed JOB_AES_HMAC structure pointer on success
1144  * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
1145  */
1146 static inline int
1147 set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
1148                 struct rte_crypto_op *op, uint8_t *digest_idx)
1149 {
1150         struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
1151         struct aesni_mb_session *session;
1152         uint32_t m_offset, oop;
1153
1154         session = get_session(qp, op);
1155         if (session == NULL) {
1156                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1157                 return -1;
1158         }
1159
1160         /* Set crypto operation */
1161         job->chain_order = session->chain_order;
1162
1163         /* Set cipher parameters */
1164         job->cipher_direction = session->cipher.direction;
1165         job->cipher_mode = session->cipher.mode;
1166
1167         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
1168
1169         /* Set authentication parameters */
1170         job->hash_alg = session->auth.algo;
1171
1172         switch (job->hash_alg) {
1173         case AES_XCBC:
1174                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
1175                 job->u.XCBC._k2 = session->auth.xcbc.k2;
1176                 job->u.XCBC._k3 = session->auth.xcbc.k3;
1177
1178                 job->aes_enc_key_expanded =
1179                                 session->cipher.expanded_aes_keys.encode;
1180                 job->aes_dec_key_expanded =
1181                                 session->cipher.expanded_aes_keys.decode;
1182                 break;
1183
1184         case AES_CCM:
1185                 job->u.CCM.aad = op->sym->aead.aad.data + 18;
1186                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1187                 job->aes_enc_key_expanded =
1188                                 session->cipher.expanded_aes_keys.encode;
1189                 job->aes_dec_key_expanded =
1190                                 session->cipher.expanded_aes_keys.decode;
1191                 break;
1192
1193         case AES_CMAC:
1194                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1195                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1196                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1197                 job->aes_enc_key_expanded =
1198                                 session->cipher.expanded_aes_keys.encode;
1199                 job->aes_dec_key_expanded =
1200                                 session->cipher.expanded_aes_keys.decode;
1201                 break;
1202
1203         case AES_GMAC:
1204                 if (session->cipher.mode == GCM) {
1205                         job->u.GCM.aad = op->sym->aead.aad.data;
1206                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1207                 } else {
1208                         /* For GMAC */
1209                         job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
1210                                         uint8_t *, op->sym->auth.data.offset);
1211                         job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
1212                         job->cipher_mode = GCM;
1213                 }
1214                 job->aes_enc_key_expanded = &session->cipher.gcm_key;
1215                 job->aes_dec_key_expanded = &session->cipher.gcm_key;
1216                 break;
1217 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
1218         case IMB_AUTH_ZUC_EIA3_BITLEN:
1219                 job->u.ZUC_EIA3._key = session->auth.zuc_auth_key;
1220                 job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1221                                                 session->auth_iv.offset);
1222                 break;
1223 #endif
1224         default:
1225                 job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
1226                 job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
1227
1228                 if (job->cipher_mode == DES3) {
1229                         job->aes_enc_key_expanded =
1230                                 session->cipher.exp_3des_keys.ks_ptr;
1231                         job->aes_dec_key_expanded =
1232                                 session->cipher.exp_3des_keys.ks_ptr;
1233                 } else {
1234                         job->aes_enc_key_expanded =
1235                                 session->cipher.expanded_aes_keys.encode;
1236                         job->aes_dec_key_expanded =
1237                                 session->cipher.expanded_aes_keys.decode;
1238                 }
1239         }
1240
1241 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
1242         if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
1243                 job->aes_enc_key_expanded = session->cipher.zuc_cipher_key;
1244                 job->aes_dec_key_expanded = session->cipher.zuc_cipher_key;
1245         }
1246 #endif
1247
1248         if (!op->sym->m_dst) {
1249                 /* in-place operation */
1250                 m_dst = m_src;
1251                 oop = 0;
1252         } else if (op->sym->m_dst == op->sym->m_src) {
1253                 /* in-place operation */
1254                 m_dst = m_src;
1255                 oop = 0;
1256         } else {
1257                 /* out-of-place operation */
1258                 m_dst = op->sym->m_dst;
1259                 oop = 1;
1260         }
1261
1262         if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
1263                         session->cipher.mode == GCM))
1264                 m_offset = op->sym->aead.data.offset;
1265         else
1266                 m_offset = op->sym->cipher.data.offset;
1267
1268         /* Set digest output location */
1269         if (job->hash_alg != NULL_HASH &&
1270                         session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1271                 job->auth_tag_output = qp->temp_digests[*digest_idx];
1272                 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1273         } else {
1274                 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
1275                                 session->cipher.mode == GCM))
1276                         job->auth_tag_output = op->sym->aead.digest.data;
1277                 else
1278                         job->auth_tag_output = op->sym->auth.digest.data;
1279
1280                 if (session->auth.req_digest_len != session->auth.gen_digest_len) {
1281                         job->auth_tag_output = qp->temp_digests[*digest_idx];
1282                         *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1283                 }
1284         }
1285         /*
1286          * Multi-buffer library current only support returning a truncated
1287          * digest length as specified in the relevant IPsec RFCs
1288          */
1289
1290         /* Set digest length */
1291         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1292
1293         /* Set IV parameters */
1294         job->iv_len_in_bytes = session->iv.length;
1295
1296         /* Data Parameters */
1297         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1298         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
1299
1300         switch (job->hash_alg) {
1301         case AES_CCM:
1302                 job->cipher_start_src_offset_in_bytes =
1303                                 op->sym->aead.data.offset;
1304                 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
1305                 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
1306                 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
1307
1308                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1309                         session->iv.offset + 1);
1310                 break;
1311
1312         case AES_GMAC:
1313                 if (session->cipher.mode == GCM) {
1314                         job->cipher_start_src_offset_in_bytes =
1315                                         op->sym->aead.data.offset;
1316                         job->hash_start_src_offset_in_bytes =
1317                                         op->sym->aead.data.offset;
1318                         job->msg_len_to_cipher_in_bytes =
1319                                         op->sym->aead.data.length;
1320                         job->msg_len_to_hash_in_bytes =
1321                                         op->sym->aead.data.length;
1322                 } else {
1323                         job->cipher_start_src_offset_in_bytes =
1324                                         op->sym->auth.data.offset;
1325                         job->hash_start_src_offset_in_bytes =
1326                                         op->sym->auth.data.offset;
1327                         job->msg_len_to_cipher_in_bytes = 0;
1328                         job->msg_len_to_hash_in_bytes = 0;
1329                 }
1330
1331                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1332                                 session->iv.offset);
1333                 break;
1334
1335         default:
1336                 job->cipher_start_src_offset_in_bytes =
1337                                 op->sym->cipher.data.offset;
1338                 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
1339
1340                 job->hash_start_src_offset_in_bytes = auth_start_offset(op,
1341                                 session, oop);
1342                 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
1343
1344                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1345                         session->iv.offset);
1346         }
1347
1348 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
1349         if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
1350                 job->msg_len_to_cipher_in_bytes >>= 3;
1351 #endif
1352
1353         /* Set user data to be crypto operation data struct */
1354         job->user_data = op;
1355
1356         return 0;
1357 }
1358
1359 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1360 /**
1361  * Process a crypto operation containing a security op and complete a
1362  * JOB_AES_HMAC job structure for submission to the multi buffer library for
1363  * processing.
1364  */
1365 static inline int
1366 set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
1367                 struct rte_crypto_op *op, uint8_t *digest_idx)
1368 {
1369         struct rte_mbuf *m_src, *m_dst;
1370         struct rte_crypto_sym_op *sym;
1371         struct aesni_mb_session *session;
1372
1373         session = get_session(qp, op);
1374         if (unlikely(session == NULL)) {
1375                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1376                 return -1;
1377         }
1378
1379         /* Only DOCSIS protocol operations supported now */
1380         if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||
1381                         session->auth.algo != IMB_AUTH_DOCSIS_CRC32) {
1382                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1383                 return -1;
1384         }
1385
1386         sym = op->sym;
1387         m_src = sym->m_src;
1388
1389         if (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {
1390                 /* in-place operation */
1391                 m_dst = m_src;
1392         } else {
1393                 /* out-of-place operation not supported */
1394                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1395                 return -ENOTSUP;
1396         }
1397
1398         /* Set crypto operation */
1399         job->chain_order = session->chain_order;
1400
1401         /* Set cipher parameters */
1402         job->cipher_direction = session->cipher.direction;
1403         job->cipher_mode = session->cipher.mode;
1404
1405         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
1406         job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
1407         job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
1408
1409         /* Set IV parameters */
1410         job->iv_len_in_bytes = session->iv.length;
1411         job->iv = (uint8_t *)op + session->iv.offset;
1412
1413         /* Set authentication parameters */
1414         job->hash_alg = session->auth.algo;
1415
1416         /* Set digest output location */
1417         job->auth_tag_output = qp->temp_digests[*digest_idx];
1418         *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1419
1420         /* Set digest length */
1421         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1422
1423         /* Set data parameters */
1424         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1425         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,
1426                                                 sym->cipher.data.offset);
1427
1428         job->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;
1429         job->msg_len_to_cipher_in_bytes = sym->cipher.data.length;
1430
1431         job->hash_start_src_offset_in_bytes = sym->auth.data.offset;
1432         job->msg_len_to_hash_in_bytes = sym->auth.data.length;
1433
1434         job->user_data = op;
1435
1436         return 0;
1437 }
1438
1439 static inline void
1440 verify_docsis_sec_crc(JOB_AES_HMAC *job, uint8_t *status)
1441 {
1442         uint16_t crc_offset;
1443         uint8_t *crc;
1444
1445         if (!job->msg_len_to_hash_in_bytes)
1446                 return;
1447
1448         crc_offset = job->hash_start_src_offset_in_bytes +
1449                         job->msg_len_to_hash_in_bytes -
1450                         job->cipher_start_src_offset_in_bytes;
1451         crc = job->dst + crc_offset;
1452
1453         /* Verify CRC (at the end of the message) */
1454         if (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0)
1455                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1456 }
1457 #endif
1458
1459 static inline void
1460 verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
1461 {
1462         /* Verify digest if required */
1463         if (memcmp(job->auth_tag_output, digest, len) != 0)
1464                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1465 }
1466
1467 static inline void
1468 generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
1469                 struct aesni_mb_session *sess)
1470 {
1471         /* No extra copy needed */
1472         if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
1473                 return;
1474
1475         /*
1476          * This can only happen for HMAC, so only digest
1477          * for authentication algos is required
1478          */
1479         memcpy(op->sym->auth.digest.data, job->auth_tag_output,
1480                         sess->auth.req_digest_len);
1481 }
1482
1483 /**
1484  * Process a completed job and return rte_mbuf which job processed
1485  *
1486  * @param qp            Queue Pair to process
1487  * @param job   JOB_AES_HMAC job to process
1488  *
1489  * @return
1490  * - Returns processed crypto operation.
1491  * - Returns NULL on invalid job
1492  */
1493 static inline struct rte_crypto_op *
1494 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
1495 {
1496         struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
1497         struct aesni_mb_session *sess = NULL;
1498
1499 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1500         uint8_t is_docsis_sec = 0;
1501
1502         if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1503                 /*
1504                  * Assuming at this point that if it's a security type op, that
1505                  * this is for DOCSIS
1506                  */
1507                 is_docsis_sec = 1;
1508                 sess = get_sec_session_private_data(op->sym->sec_session);
1509         } else
1510 #endif
1511         {
1512                 sess = get_sym_session_private_data(op->sym->session,
1513                                                 cryptodev_driver_id);
1514         }
1515
1516         if (unlikely(sess == NULL)) {
1517                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1518                 return op;
1519         }
1520
1521         if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
1522                 switch (job->status) {
1523                 case STS_COMPLETED:
1524                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1525
1526                         if (job->hash_alg == NULL_HASH)
1527                                 break;
1528
1529                         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1530                                 if (job->hash_alg == AES_CCM ||
1531                                         (job->hash_alg == AES_GMAC &&
1532                                                 sess->cipher.mode == GCM))
1533                                         verify_digest(job,
1534                                                 op->sym->aead.digest.data,
1535                                                 sess->auth.req_digest_len,
1536                                                 &op->status);
1537 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1538                                 else if (is_docsis_sec)
1539                                         verify_docsis_sec_crc(job,
1540                                                 &op->status);
1541 #endif
1542                                 else
1543                                         verify_digest(job,
1544                                                 op->sym->auth.digest.data,
1545                                                 sess->auth.req_digest_len,
1546                                                 &op->status);
1547                         } else
1548                                 generate_digest(job, op, sess);
1549                         break;
1550                 default:
1551                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1552                 }
1553         }
1554
1555         /* Free session if a session-less crypto op */
1556         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1557                 memset(sess, 0, sizeof(struct aesni_mb_session));
1558                 memset(op->sym->session, 0,
1559                         rte_cryptodev_sym_get_existing_header_session_size(
1560                                 op->sym->session));
1561                 rte_mempool_put(qp->sess_mp_priv, sess);
1562                 rte_mempool_put(qp->sess_mp, op->sym->session);
1563                 op->sym->session = NULL;
1564         }
1565
1566         return op;
1567 }
1568
1569 static inline void
1570 post_process_mb_sync_job(JOB_AES_HMAC *job)
1571 {
1572         uint32_t *st;
1573
1574         st = job->user_data;
1575         st[0] = (job->status == STS_COMPLETED) ? 0 : EBADMSG;
1576 }
1577
1578 /**
1579  * Process a completed JOB_AES_HMAC job and keep processing jobs until
1580  * get_completed_job return NULL
1581  *
1582  * @param qp            Queue Pair to process
1583  * @param job           JOB_AES_HMAC job
1584  *
1585  * @return
1586  * - Number of processed jobs
1587  */
1588 static unsigned
1589 handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
1590                 struct rte_crypto_op **ops, uint16_t nb_ops)
1591 {
1592         struct rte_crypto_op *op = NULL;
1593         unsigned processed_jobs = 0;
1594
1595         while (job != NULL) {
1596                 op = post_process_mb_job(qp, job);
1597
1598                 if (op) {
1599                         ops[processed_jobs++] = op;
1600                         qp->stats.dequeued_count++;
1601                 } else {
1602                         qp->stats.dequeue_err_count++;
1603                         break;
1604                 }
1605                 if (processed_jobs == nb_ops)
1606                         break;
1607
1608                 job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
1609         }
1610
1611         return processed_jobs;
1612 }
1613
1614 static inline uint32_t
1615 handle_completed_sync_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr)
1616 {
1617         uint32_t i;
1618
1619         for (i = 0; job != NULL; i++, job = IMB_GET_COMPLETED_JOB(mb_mgr))
1620                 post_process_mb_sync_job(job);
1621
1622         return i;
1623 }
1624
1625 static inline uint32_t
1626 flush_mb_sync_mgr(MB_MGR *mb_mgr)
1627 {
1628         JOB_AES_HMAC *job;
1629
1630         job = IMB_FLUSH_JOB(mb_mgr);
1631         return handle_completed_sync_jobs(job, mb_mgr);
1632 }
1633
1634 static inline uint16_t
1635 flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
1636                 uint16_t nb_ops)
1637 {
1638         int processed_ops = 0;
1639
1640         /* Flush the remaining jobs */
1641         JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
1642
1643         if (job)
1644                 processed_ops += handle_completed_jobs(qp, job,
1645                                 &ops[processed_ops], nb_ops - processed_ops);
1646
1647         return processed_ops;
1648 }
1649
1650 static inline JOB_AES_HMAC *
1651 set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
1652 {
1653         job->chain_order = HASH_CIPHER;
1654         job->cipher_mode = NULL_CIPHER;
1655         job->hash_alg = NULL_HASH;
1656         job->cipher_direction = DECRYPT;
1657
1658         /* Set user data to be crypto operation data struct */
1659         job->user_data = op;
1660
1661         return job;
1662 }
1663
1664 static uint16_t
1665 aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1666                 uint16_t nb_ops)
1667 {
1668         struct aesni_mb_qp *qp = queue_pair;
1669
1670         struct rte_crypto_op *op;
1671         JOB_AES_HMAC *job;
1672
1673         int retval, processed_jobs = 0;
1674
1675         if (unlikely(nb_ops == 0))
1676                 return 0;
1677
1678         uint8_t digest_idx = qp->digest_idx;
1679         do {
1680                 /* Get next free mb job struct from mb manager */
1681                 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1682                 if (unlikely(job == NULL)) {
1683                         /* if no free mb job structs we need to flush mb_mgr */
1684                         processed_jobs += flush_mb_mgr(qp,
1685                                         &ops[processed_jobs],
1686                                         nb_ops - processed_jobs);
1687
1688                         if (nb_ops == processed_jobs)
1689                                 break;
1690
1691                         job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1692                 }
1693
1694                 /*
1695                  * Get next operation to process from ingress queue.
1696                  * There is no need to return the job to the MB_MGR
1697                  * if there are no more operations to process, since the MB_MGR
1698                  * can use that pointer again in next get_next calls.
1699                  */
1700                 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1701                 if (retval < 0)
1702                         break;
1703
1704 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1705                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1706                         retval = set_sec_mb_job_params(job, qp, op,
1707                                                 &digest_idx);
1708                 else
1709 #endif
1710                         retval = set_mb_job_params(job, qp, op, &digest_idx);
1711
1712                 if (unlikely(retval != 0)) {
1713                         qp->stats.dequeue_err_count++;
1714                         set_job_null_op(job, op);
1715                 }
1716
1717                 /* Submit job to multi-buffer for processing */
1718 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1719                 job = IMB_SUBMIT_JOB(qp->mb_mgr);
1720 #else
1721                 job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
1722 #endif
1723                 /*
1724                  * If submit returns a processed job then handle it,
1725                  * before submitting subsequent jobs
1726                  */
1727                 if (job)
1728                         processed_jobs += handle_completed_jobs(qp, job,
1729                                         &ops[processed_jobs],
1730                                         nb_ops - processed_jobs);
1731
1732         } while (processed_jobs < nb_ops);
1733
1734         qp->digest_idx = digest_idx;
1735
1736         if (processed_jobs < 1)
1737                 processed_jobs += flush_mb_mgr(qp,
1738                                 &ops[processed_jobs],
1739                                 nb_ops - processed_jobs);
1740
1741         return processed_jobs;
1742 }
1743
1744 static MB_MGR *
1745 alloc_init_mb_mgr(enum aesni_mb_vector_mode vector_mode)
1746 {
1747         MB_MGR *mb_mgr = alloc_mb_mgr(0);
1748         if (mb_mgr == NULL)
1749                 return NULL;
1750
1751         switch (vector_mode) {
1752         case RTE_AESNI_MB_SSE:
1753                 init_mb_mgr_sse(mb_mgr);
1754                 break;
1755         case RTE_AESNI_MB_AVX:
1756                 init_mb_mgr_avx(mb_mgr);
1757                 break;
1758         case RTE_AESNI_MB_AVX2:
1759                 init_mb_mgr_avx2(mb_mgr);
1760                 break;
1761         case RTE_AESNI_MB_AVX512:
1762                 init_mb_mgr_avx512(mb_mgr);
1763                 break;
1764         default:
1765                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
1766                 free_mb_mgr(mb_mgr);
1767                 return NULL;
1768         }
1769
1770         return mb_mgr;
1771 }
1772
1773 static inline void
1774 aesni_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
1775 {
1776         uint32_t i;
1777
1778         for (i = 0; i != vec->num; ++i)
1779                 vec->status[i] = err;
1780 }
1781
1782 static inline int
1783 check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)
1784 {
1785         /* no multi-seg support with current AESNI-MB PMD */
1786         if (sgl->num != 1)
1787                 return ENOTSUP;
1788         else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)
1789                 return EINVAL;
1790         return 0;
1791 }
1792
1793 static inline JOB_AES_HMAC *
1794 submit_sync_job(MB_MGR *mb_mgr)
1795 {
1796 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1797         return IMB_SUBMIT_JOB(mb_mgr);
1798 #else
1799         return IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
1800 #endif
1801 }
1802
1803 static inline uint32_t
1804 generate_sync_dgst(struct rte_crypto_sym_vec *vec,
1805         const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1806 {
1807         uint32_t i, k;
1808
1809         for (i = 0, k = 0; i != vec->num; i++) {
1810                 if (vec->status[i] == 0) {
1811                         memcpy(vec->digest[i], dgst[i], len);
1812                         k++;
1813                 }
1814         }
1815
1816         return k;
1817 }
1818
1819 static inline uint32_t
1820 verify_sync_dgst(struct rte_crypto_sym_vec *vec,
1821         const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1822 {
1823         uint32_t i, k;
1824
1825         for (i = 0, k = 0; i != vec->num; i++) {
1826                 if (vec->status[i] == 0) {
1827                         if (memcmp(vec->digest[i], dgst[i], len) != 0)
1828                                 vec->status[i] = EBADMSG;
1829                         else
1830                                 k++;
1831                 }
1832         }
1833
1834         return k;
1835 }
1836
1837 uint32_t
1838 aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
1839         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
1840         struct rte_crypto_sym_vec *vec)
1841 {
1842         int32_t ret;
1843         uint32_t i, j, k, len;
1844         void *buf;
1845         JOB_AES_HMAC *job;
1846         MB_MGR *mb_mgr;
1847         struct aesni_mb_private *priv;
1848         struct aesni_mb_session *s;
1849         uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];
1850
1851         s = get_sym_session_private_data(sess, dev->driver_id);
1852         if (s == NULL) {
1853                 aesni_mb_fill_error_code(vec, EINVAL);
1854                 return 0;
1855         }
1856
1857         /* get per-thread MB MGR, create one if needed */
1858         mb_mgr = RTE_PER_LCORE(sync_mb_mgr);
1859         if (mb_mgr == NULL) {
1860
1861                 priv = dev->data->dev_private;
1862                 mb_mgr = alloc_init_mb_mgr(priv->vector_mode);
1863                 if (mb_mgr == NULL) {
1864                         aesni_mb_fill_error_code(vec, ENOMEM);
1865                         return 0;
1866                 }
1867                 RTE_PER_LCORE(sync_mb_mgr) = mb_mgr;
1868         }
1869
1870         for (i = 0, j = 0, k = 0; i != vec->num; i++) {
1871
1872
1873                 ret = check_crypto_sgl(sofs, vec->sgl + i);
1874                 if (ret != 0) {
1875                         vec->status[i] = ret;
1876                         continue;
1877                 }
1878
1879                 buf = vec->sgl[i].vec[0].base;
1880                 len = vec->sgl[i].vec[0].len;
1881
1882                 job = IMB_GET_NEXT_JOB(mb_mgr);
1883                 if (job == NULL) {
1884                         k += flush_mb_sync_mgr(mb_mgr);
1885                         job = IMB_GET_NEXT_JOB(mb_mgr);
1886                         RTE_ASSERT(job != NULL);
1887                 }
1888
1889                 /* Submit job for processing */
1890                 set_cpu_mb_job_params(job, s, sofs, buf, len,
1891                         vec->iv[i], vec->aad[i], tmp_dgst[i],
1892                         &vec->status[i]);
1893                 job = submit_sync_job(mb_mgr);
1894                 j++;
1895
1896                 /* handle completed jobs */
1897                 k += handle_completed_sync_jobs(job, mb_mgr);
1898         }
1899
1900         /* flush remaining jobs */
1901         while (k != j)
1902                 k += flush_mb_sync_mgr(mb_mgr);
1903
1904         /* finish processing for successful jobs: check/update digest */
1905         if (k != 0) {
1906                 if (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1907                         k = verify_sync_dgst(vec,
1908                                 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1909                                 s->auth.req_digest_len);
1910                 else
1911                         k = generate_sync_dgst(vec,
1912                                 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1913                                 s->auth.req_digest_len);
1914         }
1915
1916         return k;
1917 }
1918
1919 static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
1920
1921 static uint64_t
1922 vec_mode_to_flags(enum aesni_mb_vector_mode mode)
1923 {
1924         switch (mode) {
1925         case RTE_AESNI_MB_SSE:
1926                 return RTE_CRYPTODEV_FF_CPU_SSE;
1927         case RTE_AESNI_MB_AVX:
1928                 return RTE_CRYPTODEV_FF_CPU_AVX;
1929         case RTE_AESNI_MB_AVX2:
1930                 return RTE_CRYPTODEV_FF_CPU_AVX2;
1931         case RTE_AESNI_MB_AVX512:
1932                 return RTE_CRYPTODEV_FF_CPU_AVX512;
1933         default:
1934                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", mode);
1935                 return 0;
1936         }
1937 }
1938
1939 static int
1940 cryptodev_aesni_mb_create(const char *name,
1941                         struct rte_vdev_device *vdev,
1942                         struct rte_cryptodev_pmd_init_params *init_params)
1943 {
1944         struct rte_cryptodev *dev;
1945         struct aesni_mb_private *internals;
1946         enum aesni_mb_vector_mode vector_mode;
1947         MB_MGR *mb_mgr;
1948
1949         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1950         if (dev == NULL) {
1951                 AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
1952                 return -ENODEV;
1953         }
1954
1955         /* Check CPU for supported vector instruction set */
1956         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
1957                 vector_mode = RTE_AESNI_MB_AVX512;
1958         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
1959                 vector_mode = RTE_AESNI_MB_AVX2;
1960         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
1961                 vector_mode = RTE_AESNI_MB_AVX;
1962         else
1963                 vector_mode = RTE_AESNI_MB_SSE;
1964
1965         dev->driver_id = cryptodev_driver_id;
1966         dev->dev_ops = rte_aesni_mb_pmd_ops;
1967
1968         /* register rx/tx burst functions for data path */
1969         dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
1970         dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
1971
1972         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1973                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1974                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
1975                         RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
1976                         RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
1977                         RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
1978
1979 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1980         struct rte_security_ctx *security_instance;
1981         security_instance = rte_malloc("aesni_mb_sec",
1982                                 sizeof(struct rte_security_ctx),
1983                                 RTE_CACHE_LINE_SIZE);
1984         if (security_instance == NULL) {
1985                 AESNI_MB_LOG(ERR, "rte_security_ctx memory alloc failed");
1986                 rte_cryptodev_pmd_destroy(dev);
1987                 return -ENOMEM;
1988         }
1989
1990         security_instance->device = (void *)dev;
1991         security_instance->ops = rte_aesni_mb_pmd_sec_ops;
1992         security_instance->sess_cnt = 0;
1993         dev->security_ctx = security_instance;
1994         dev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
1995 #endif
1996
1997         /* Check CPU for support for AES instruction set */
1998         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
1999                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
2000         else
2001                 AESNI_MB_LOG(WARNING, "AES instructions not supported by CPU");
2002
2003         dev->feature_flags |= vec_mode_to_flags(vector_mode);
2004
2005         mb_mgr = alloc_init_mb_mgr(vector_mode);
2006         if (mb_mgr == NULL) {
2007 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
2008                 rte_free(dev->security_ctx);
2009                 dev->security_ctx = NULL;
2010 #endif
2011                 rte_cryptodev_pmd_destroy(dev);
2012                 return -ENOMEM;
2013         }
2014
2015         /* Set vector instructions mode supported */
2016         internals = dev->data->dev_private;
2017
2018         internals->vector_mode = vector_mode;
2019         internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
2020         internals->mb_mgr = mb_mgr;
2021
2022         AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
2023                         imb_get_version_str());
2024         return 0;
2025 }
2026
2027 static int
2028 cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
2029 {
2030         struct rte_cryptodev_pmd_init_params init_params = {
2031                 "",
2032                 sizeof(struct aesni_mb_private),
2033                 rte_socket_id(),
2034                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2035         };
2036         const char *name, *args;
2037         int retval;
2038
2039         name = rte_vdev_device_name(vdev);
2040         if (name == NULL)
2041                 return -EINVAL;
2042
2043         args = rte_vdev_device_args(vdev);
2044
2045         retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
2046         if (retval) {
2047                 AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
2048                                 args);
2049                 return -EINVAL;
2050         }
2051
2052         return cryptodev_aesni_mb_create(name, vdev, &init_params);
2053 }
2054
2055 static int
2056 cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
2057 {
2058         struct rte_cryptodev *cryptodev;
2059         struct aesni_mb_private *internals;
2060         const char *name;
2061
2062         name = rte_vdev_device_name(vdev);
2063         if (name == NULL)
2064                 return -EINVAL;
2065
2066         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2067         if (cryptodev == NULL)
2068                 return -ENODEV;
2069
2070         internals = cryptodev->data->dev_private;
2071
2072         free_mb_mgr(internals->mb_mgr);
2073         if (RTE_PER_LCORE(sync_mb_mgr)) {
2074                 free_mb_mgr(RTE_PER_LCORE(sync_mb_mgr));
2075                 RTE_PER_LCORE(sync_mb_mgr) = NULL;
2076         }
2077
2078 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
2079         rte_free(cryptodev->security_ctx);
2080         cryptodev->security_ctx = NULL;
2081 #endif
2082
2083         return rte_cryptodev_pmd_destroy(cryptodev);
2084 }
2085
2086 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
2087         .probe = cryptodev_aesni_mb_probe,
2088         .remove = cryptodev_aesni_mb_remove
2089 };
2090
2091 static struct cryptodev_driver aesni_mb_crypto_drv;
2092
2093 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
2094 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
2095 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
2096         "max_nb_queue_pairs=<int> "
2097         "socket_id=<int>");
2098 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
2099                 cryptodev_aesni_mb_pmd_drv.driver,
2100                 cryptodev_driver_id);
2101 RTE_LOG_REGISTER(aesni_mb_logtype_driver, pmd.crypto.aesni_mb, NOTICE);