drivers/crypto/ipsec_mb/pmd_aesni_mb.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2021 Intel Corporation
   3  */
   4
   5 #include "pmd_aesni_mb_priv.h"
   6
   7 /**
   8  * Calculate the authentication pre-computes
   9  *
  10  * @param one_block_hash        Function pointer
  11  *                              to calculate digest on ipad/opad
  12  * @param ipad                  Inner pad output byte array
  13  * @param opad                  Outer pad output byte array
  14  * @param hkey                  Authentication key
  15  * @param hkey_len              Authentication key length
  16  * @param blocksize             Block size of selected hash algo
  17  */
  18 static void
  19 calculate_auth_precomputes(hash_one_block_t one_block_hash,
  20                 uint8_t *ipad, uint8_t *opad,
  21                 const uint8_t *hkey, uint16_t hkey_len,
  22                 uint16_t blocksize)
  23 {
  24         uint32_t i, length;
  25
  26         uint8_t ipad_buf[blocksize] __rte_aligned(16);
  27         uint8_t opad_buf[blocksize] __rte_aligned(16);
  28
  29         /* Setup inner and outer pads */
  30         memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
  31         memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
  32
  33         /* XOR hash key with inner and outer pads */
  34         length = hkey_len > blocksize ? blocksize : hkey_len;
  35
  36         for (i = 0; i < length; i++) {
  37                 ipad_buf[i] ^= hkey[i];
  38                 opad_buf[i] ^= hkey[i];
  39         }
  40
  41         /* Compute partial hashes */
  42         (*one_block_hash)(ipad_buf, ipad);
  43         (*one_block_hash)(opad_buf, opad);
  44
  45         /* Clean up stack */
  46         memset(ipad_buf, 0, blocksize);
  47         memset(opad_buf, 0, blocksize);
  48 }
  49
  50 static inline int
  51 is_aead_algo(IMB_HASH_ALG hash_alg, IMB_CIPHER_MODE cipher_mode)
  52 {
  53         return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 ||
  54                 hash_alg == IMB_AUTH_AES_CCM ||
  55                 (hash_alg == IMB_AUTH_AES_GMAC &&
  56                 cipher_mode == IMB_CIPHER_GCM));
  57 }
  58
  59 /** Set session authentication parameters */
  60 static int
  61 aesni_mb_set_session_auth_parameters(const IMB_MGR *mb_mgr,
  62                 struct aesni_mb_session *sess,
  63                 const struct rte_crypto_sym_xform *xform)
  64 {
  65         hash_one_block_t hash_oneblock_fn = NULL;
  66         unsigned int key_larger_block_size = 0;
  67         uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
  68         uint32_t auth_precompute = 1;
  69
  70         if (xform == NULL) {
  71                 sess->auth.algo = IMB_AUTH_NULL;
  72                 return 0;
  73         }
  74
  75         if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
  76                 IPSEC_MB_LOG(ERR, "Crypto xform struct not of type auth");
  77                 return -1;
  78         }
  79
  80         /* Set IV parameters */
  81         sess->auth_iv.offset = xform->auth.iv.offset;
  82         sess->auth_iv.length = xform->auth.iv.length;
  83
  84         /* Set the request digest size */
  85         sess->auth.req_digest_len = xform->auth.digest_length;
  86
  87         /* Select auth generate/verify */
  88         sess->auth.operation = xform->auth.op;
  89
  90         /* Set Authentication Parameters */
  91         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
  92                 sess->auth.algo = IMB_AUTH_AES_XCBC;
  93
  94                 uint16_t xcbc_mac_digest_len =
  95                         get_truncated_digest_byte_length(IMB_AUTH_AES_XCBC);
  96                 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
  97                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
  98                         return -EINVAL;
  99                 }
 100                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 101
 102                 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
 103                                 sess->auth.xcbc.k1_expanded,
 104                                 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
 105                 return 0;
 106         }
 107
 108         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
 109                 uint32_t dust[4*15];
 110
 111                 sess->auth.algo = IMB_AUTH_AES_CMAC;
 112
 113                 uint16_t cmac_digest_len =
 114                                 get_digest_byte_length(IMB_AUTH_AES_CMAC);
 115
 116                 if (sess->auth.req_digest_len > cmac_digest_len) {
 117                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 118                         return -EINVAL;
 119                 }
 120                 /*
 121                  * Multi-buffer lib supports digest sizes from 4 to 16 bytes
 122                  * in version 0.50 and sizes of 12 and 16 bytes,
 123                  * in version 0.49.
 124                  * If size requested is different, generate the full digest
 125                  * (16 bytes) in a temporary location and then memcpy
 126                  * the requested number of bytes.
 127                  */
 128                 if (sess->auth.req_digest_len < 4)
 129                         sess->auth.gen_digest_len = cmac_digest_len;
 130                 else
 131                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 132
 133                 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
 134                                 sess->auth.cmac.expkey, dust);
 135                 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
 136                                 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
 137                 return 0;
 138         }
 139
 140         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 141                 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 142                         sess->cipher.direction = IMB_DIR_ENCRYPT;
 143                         sess->chain_order = IMB_ORDER_CIPHER_HASH;
 144                 } else
 145                         sess->cipher.direction = IMB_DIR_DECRYPT;
 146
 147                 sess->auth.algo = IMB_AUTH_AES_GMAC;
 148                 if (sess->auth.req_digest_len >
 149                         get_digest_byte_length(IMB_AUTH_AES_GMAC)) {
 150                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 151                         return -EINVAL;
 152                 }
 153                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 154                 sess->iv.length = xform->auth.iv.length;
 155                 sess->iv.offset = xform->auth.iv.offset;
 156
 157                 switch (xform->auth.key.length) {
 158                 case IMB_KEY_128_BYTES:
 159                         IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
 160                                 &sess->cipher.gcm_key);
 161                         sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
 162                         break;
 163                 case IMB_KEY_192_BYTES:
 164                         IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
 165                                 &sess->cipher.gcm_key);
 166                         sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
 167                         break;
 168                 case IMB_KEY_256_BYTES:
 169                         IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
 170                                 &sess->cipher.gcm_key);
 171                         sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
 172                         break;
 173                 default:
 174                         IPSEC_MB_LOG(ERR, "Invalid authentication key length\n");
 175                         return -EINVAL;
 176                 }
 177
 178                 return 0;
 179         }
 180
 181         if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 182                 sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;
 183                 uint16_t zuc_eia3_digest_len =
 184                         get_truncated_digest_byte_length(
 185                                                 IMB_AUTH_ZUC_EIA3_BITLEN);
 186                 if (sess->auth.req_digest_len != zuc_eia3_digest_len) {
 187                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 188                         return -EINVAL;
 189                 }
 190                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 191
 192                 memcpy(sess->auth.zuc_auth_key, xform->auth.key.data, 16);
 193                 return 0;
 194         } else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 195                 sess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN;
 196                 uint16_t snow3g_uia2_digest_len =
 197                         get_truncated_digest_byte_length(
 198                                                 IMB_AUTH_SNOW3G_UIA2_BITLEN);
 199                 if (sess->auth.req_digest_len != snow3g_uia2_digest_len) {
 200                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 201                         return -EINVAL;
 202                 }
 203                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 204
 205                 IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->auth.key.data,
 206                                         &sess->auth.pKeySched_snow3g_auth);
 207                 return 0;
 208         } else if (xform->auth.algo == RTE_CRYPTO_AUTH_KASUMI_F9) {
 209                 sess->auth.algo = IMB_AUTH_KASUMI_UIA1;
 210                 uint16_t kasumi_f9_digest_len =
 211                         get_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1);
 212                 if (sess->auth.req_digest_len != kasumi_f9_digest_len) {
 213                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 214                         return -EINVAL;
 215                 }
 216                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 217
 218                 IMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr, xform->auth.key.data,
 219                                         &sess->auth.pKeySched_kasumi_auth);
 220                 return 0;
 221         }
 222
 223         switch (xform->auth.algo) {
 224         case RTE_CRYPTO_AUTH_MD5_HMAC:
 225                 sess->auth.algo = IMB_AUTH_MD5;
 226                 hash_oneblock_fn = mb_mgr->md5_one_block;
 227                 break;
 228         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 229                 sess->auth.algo = IMB_AUTH_HMAC_SHA_1;
 230                 hash_oneblock_fn = mb_mgr->sha1_one_block;
 231                 if (xform->auth.key.length > get_auth_algo_blocksize(
 232                                 IMB_AUTH_HMAC_SHA_1)) {
 233                         IMB_SHA1(mb_mgr,
 234                                 xform->auth.key.data,
 235                                 xform->auth.key.length,
 236                                 hashed_key);
 237                         key_larger_block_size = 1;
 238                 }
 239                 break;
 240         case RTE_CRYPTO_AUTH_SHA1:
 241                 sess->auth.algo = IMB_AUTH_SHA_1;
 242                 auth_precompute = 0;
 243                 break;
 244         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 245                 sess->auth.algo = IMB_AUTH_HMAC_SHA_224;
 246                 hash_oneblock_fn = mb_mgr->sha224_one_block;
 247                 if (xform->auth.key.length > get_auth_algo_blocksize(
 248                                 IMB_AUTH_HMAC_SHA_224)) {
 249                         IMB_SHA224(mb_mgr,
 250                                 xform->auth.key.data,
 251                                 xform->auth.key.length,
 252                                 hashed_key);
 253                         key_larger_block_size = 1;
 254                 }
 255                 break;
 256         case RTE_CRYPTO_AUTH_SHA224:
 257                 sess->auth.algo = IMB_AUTH_SHA_224;
 258                 auth_precompute = 0;
 259                 break;
 260         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 261                 sess->auth.algo = IMB_AUTH_HMAC_SHA_256;
 262                 hash_oneblock_fn = mb_mgr->sha256_one_block;
 263                 if (xform->auth.key.length > get_auth_algo_blocksize(
 264                                 IMB_AUTH_HMAC_SHA_256)) {
 265                         IMB_SHA256(mb_mgr,
 266                                 xform->auth.key.data,
 267                                 xform->auth.key.length,
 268                                 hashed_key);
 269                         key_larger_block_size = 1;
 270                 }
 271                 break;
 272         case RTE_CRYPTO_AUTH_SHA256:
 273                 sess->auth.algo = IMB_AUTH_SHA_256;
 274                 auth_precompute = 0;
 275                 break;
 276         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 277                 sess->auth.algo = IMB_AUTH_HMAC_SHA_384;
 278                 hash_oneblock_fn = mb_mgr->sha384_one_block;
 279                 if (xform->auth.key.length > get_auth_algo_blocksize(
 280                                 IMB_AUTH_HMAC_SHA_384)) {
 281                         IMB_SHA384(mb_mgr,
 282                                 xform->auth.key.data,
 283                                 xform->auth.key.length,
 284                                 hashed_key);
 285                         key_larger_block_size = 1;
 286                 }
 287                 break;
 288         case RTE_CRYPTO_AUTH_SHA384:
 289                 sess->auth.algo = IMB_AUTH_SHA_384;
 290                 auth_precompute = 0;
 291                 break;
 292         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 293                 sess->auth.algo = IMB_AUTH_HMAC_SHA_512;
 294                 hash_oneblock_fn = mb_mgr->sha512_one_block;
 295                 if (xform->auth.key.length > get_auth_algo_blocksize(
 296                                 IMB_AUTH_HMAC_SHA_512)) {
 297                         IMB_SHA512(mb_mgr,
 298                                 xform->auth.key.data,
 299                                 xform->auth.key.length,
 300                                 hashed_key);
 301                         key_larger_block_size = 1;
 302                 }
 303                 break;
 304         case RTE_CRYPTO_AUTH_SHA512:
 305                 sess->auth.algo = IMB_AUTH_SHA_512;
 306                 auth_precompute = 0;
 307                 break;
 308         default:
 309                 IPSEC_MB_LOG(ERR,
 310                         "Unsupported authentication algorithm selection");
 311                 return -ENOTSUP;
 312         }
 313         uint16_t trunc_digest_size =
 314                         get_truncated_digest_byte_length(sess->auth.algo);
 315         uint16_t full_digest_size =
 316                         get_digest_byte_length(sess->auth.algo);
 317
 318         if (sess->auth.req_digest_len > full_digest_size ||
 319                         sess->auth.req_digest_len == 0) {
 320                 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 321                 return -EINVAL;
 322         }
 323
 324         if (sess->auth.req_digest_len != trunc_digest_size &&
 325                         sess->auth.req_digest_len != full_digest_size)
 326                 sess->auth.gen_digest_len = full_digest_size;
 327         else
 328                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 329
 330         /* Plain SHA does not require precompute key */
 331         if (auth_precompute == 0)
 332                 return 0;
 333
 334         /* Calculate Authentication precomputes */
 335         if (key_larger_block_size) {
 336                 calculate_auth_precomputes(hash_oneblock_fn,
 337                         sess->auth.pads.inner, sess->auth.pads.outer,
 338                         hashed_key,
 339                         xform->auth.key.length,
 340                         get_auth_algo_blocksize(sess->auth.algo));
 341         } else {
 342                 calculate_auth_precomputes(hash_oneblock_fn,
 343                         sess->auth.pads.inner, sess->auth.pads.outer,
 344                         xform->auth.key.data,
 345                         xform->auth.key.length,
 346                         get_auth_algo_blocksize(sess->auth.algo));
 347         }
 348
 349         return 0;
 350 }
 351
 352 /** Set session cipher parameters */
 353 static int
 354 aesni_mb_set_session_cipher_parameters(const IMB_MGR *mb_mgr,
 355                 struct aesni_mb_session *sess,
 356                 const struct rte_crypto_sym_xform *xform)
 357 {
 358         uint8_t is_aes = 0;
 359         uint8_t is_3DES = 0;
 360         uint8_t is_docsis = 0;
 361         uint8_t is_zuc = 0;
 362         uint8_t is_snow3g = 0;
 363         uint8_t is_kasumi = 0;
 364
 365         if (xform == NULL) {
 366                 sess->cipher.mode = IMB_CIPHER_NULL;
 367                 return 0;
 368         }
 369
 370         if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
 371                 IPSEC_MB_LOG(ERR, "Crypto xform struct not of type cipher");
 372                 return -EINVAL;
 373         }
 374
 375         /* Select cipher direction */
 376         switch (xform->cipher.op) {
 377         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
 378                 sess->cipher.direction = IMB_DIR_ENCRYPT;
 379                 break;
 380         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
 381                 sess->cipher.direction = IMB_DIR_DECRYPT;
 382                 break;
 383         default:
 384                 IPSEC_MB_LOG(ERR, "Invalid cipher operation parameter");
 385                 return -EINVAL;
 386         }
 387
 388         /* Select cipher mode */
 389         switch (xform->cipher.algo) {
 390         case RTE_CRYPTO_CIPHER_AES_CBC:
 391                 sess->cipher.mode = IMB_CIPHER_CBC;
 392                 is_aes = 1;
 393                 break;
 394         case RTE_CRYPTO_CIPHER_AES_CTR:
 395                 sess->cipher.mode = IMB_CIPHER_CNTR;
 396                 is_aes = 1;
 397                 break;
 398         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 399                 sess->cipher.mode = IMB_CIPHER_DOCSIS_SEC_BPI;
 400                 is_docsis = 1;
 401                 break;
 402         case RTE_CRYPTO_CIPHER_DES_CBC:
 403                 sess->cipher.mode = IMB_CIPHER_DES;
 404                 break;
 405         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 406                 sess->cipher.mode = IMB_CIPHER_DOCSIS_DES;
 407                 break;
 408         case RTE_CRYPTO_CIPHER_3DES_CBC:
 409                 sess->cipher.mode = IMB_CIPHER_DES3;
 410                 is_3DES = 1;
 411                 break;
 412         case RTE_CRYPTO_CIPHER_AES_ECB:
 413                 sess->cipher.mode = IMB_CIPHER_ECB;
 414                 is_aes = 1;
 415                 break;
 416         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 417                 sess->cipher.mode = IMB_CIPHER_ZUC_EEA3;
 418                 is_zuc = 1;
 419                 break;
 420         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 421                 sess->cipher.mode = IMB_CIPHER_SNOW3G_UEA2_BITLEN;
 422                 is_snow3g = 1;
 423                 break;
 424         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 425                 sess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN;
 426                 is_kasumi = 1;
 427                 break;
 428         default:
 429                 IPSEC_MB_LOG(ERR, "Unsupported cipher mode parameter");
 430                 return -ENOTSUP;
 431         }
 432
 433         /* Set IV parameters */
 434         sess->iv.offset = xform->cipher.iv.offset;
 435         sess->iv.length = xform->cipher.iv.length;
 436
 437         /* Check key length and choose key expansion function for AES */
 438         if (is_aes) {
 439                 switch (xform->cipher.key.length) {
 440                 case IMB_KEY_128_BYTES:
 441                         sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
 442                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 443                                         sess->cipher.expanded_aes_keys.encode,
 444                                         sess->cipher.expanded_aes_keys.decode);
 445                         break;
 446                 case IMB_KEY_192_BYTES:
 447                         sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
 448                         IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
 449                                         sess->cipher.expanded_aes_keys.encode,
 450                                         sess->cipher.expanded_aes_keys.decode);
 451                         break;
 452                 case IMB_KEY_256_BYTES:
 453                         sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
 454                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 455                                         sess->cipher.expanded_aes_keys.encode,
 456                                         sess->cipher.expanded_aes_keys.decode);
 457                         break;
 458                 default:
 459                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 460                         return -EINVAL;
 461                 }
 462         } else if (is_docsis) {
 463                 switch (xform->cipher.key.length) {
 464                 case IMB_KEY_128_BYTES:
 465                         sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
 466                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 467                                         sess->cipher.expanded_aes_keys.encode,
 468                                         sess->cipher.expanded_aes_keys.decode);
 469                         break;
 470                 case IMB_KEY_256_BYTES:
 471                         sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
 472                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 473                                         sess->cipher.expanded_aes_keys.encode,
 474                                         sess->cipher.expanded_aes_keys.decode);
 475                         break;
 476                 default:
 477                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 478                         return -EINVAL;
 479                 }
 480         } else if (is_3DES) {
 481                 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
 482                                 sess->cipher.exp_3des_keys.key[1],
 483                                 sess->cipher.exp_3des_keys.key[2]};
 484
 485                 switch (xform->cipher.key.length) {
 486                 case  24:
 487                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 488                                         xform->cipher.key.data);
 489                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 490                                         xform->cipher.key.data + 8);
 491                         IMB_DES_KEYSCHED(mb_mgr, keys[2],
 492                                         xform->cipher.key.data + 16);
 493
 494                         /* Initialize keys - 24 bytes: [K1-K2-K3] */
 495                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 496                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 497                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
 498                         break;
 499                 case 16:
 500                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 501                                         xform->cipher.key.data);
 502                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 503                                         xform->cipher.key.data + 8);
 504                         /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
 505                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 506                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 507                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 508                         break;
 509                 case 8:
 510                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 511                                         xform->cipher.key.data);
 512
 513                         /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
 514                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 515                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
 516                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 517                         break;
 518                 default:
 519                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 520                         return -EINVAL;
 521                 }
 522
 523                 sess->cipher.key_length_in_bytes = 24;
 524         } else if (is_zuc) {
 525                 if (xform->cipher.key.length != 16) {
 526                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 527                         return -EINVAL;
 528                 }
 529                 sess->cipher.key_length_in_bytes = 16;
 530                 memcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data,
 531                         16);
 532         } else if (is_snow3g) {
 533                 if (xform->cipher.key.length != 16) {
 534                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 535                         return -EINVAL;
 536                 }
 537                 sess->cipher.key_length_in_bytes = 16;
 538                 IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->cipher.key.data,
 539                                         &sess->cipher.pKeySched_snow3g_cipher);
 540         } else if (is_kasumi) {
 541                 if (xform->cipher.key.length != 16) {
 542                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 543                         return -EINVAL;
 544                 }
 545                 sess->cipher.key_length_in_bytes = 16;
 546                 IMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data,
 547                                         &sess->cipher.pKeySched_kasumi_cipher);
 548         } else {
 549                 if (xform->cipher.key.length != 8) {
 550                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 551                         return -EINVAL;
 552                 }
 553                 sess->cipher.key_length_in_bytes = 8;
 554
 555                 IMB_DES_KEYSCHED(mb_mgr,
 556                         (uint64_t *)sess->cipher.expanded_aes_keys.encode,
 557                                 xform->cipher.key.data);
 558                 IMB_DES_KEYSCHED(mb_mgr,
 559                         (uint64_t *)sess->cipher.expanded_aes_keys.decode,
 560                                 xform->cipher.key.data);
 561         }
 562
 563         return 0;
 564 }
 565
 566 static int
 567 aesni_mb_set_session_aead_parameters(const IMB_MGR *mb_mgr,
 568                 struct aesni_mb_session *sess,
 569                 const struct rte_crypto_sym_xform *xform)
 570 {
 571         switch (xform->aead.op) {
 572         case RTE_CRYPTO_AEAD_OP_ENCRYPT:
 573                 sess->cipher.direction = IMB_DIR_ENCRYPT;
 574                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 575                 break;
 576         case RTE_CRYPTO_AEAD_OP_DECRYPT:
 577                 sess->cipher.direction = IMB_DIR_DECRYPT;
 578                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 579                 break;
 580         default:
 581                 IPSEC_MB_LOG(ERR, "Invalid aead operation parameter");
 582                 return -EINVAL;
 583         }
 584
 585         /* Set IV parameters */
 586         sess->iv.offset = xform->aead.iv.offset;
 587         sess->iv.length = xform->aead.iv.length;
 588
 589         /* Set digest sizes */
 590         sess->auth.req_digest_len = xform->aead.digest_length;
 591         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 592
 593         switch (xform->aead.algo) {
 594         case RTE_CRYPTO_AEAD_AES_CCM:
 595                 sess->cipher.mode = IMB_CIPHER_CCM;
 596                 sess->auth.algo = IMB_AUTH_AES_CCM;
 597
 598                 /* Check key length and choose key expansion function for AES */
 599                 switch (xform->aead.key.length) {
 600                 case IMB_KEY_128_BYTES:
 601                         sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
 602                         IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
 603                                         sess->cipher.expanded_aes_keys.encode,
 604                                         sess->cipher.expanded_aes_keys.decode);
 605                         break;
 606                 case IMB_KEY_256_BYTES:
 607                         sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
 608                         IMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data,
 609                                         sess->cipher.expanded_aes_keys.encode,
 610                                         sess->cipher.expanded_aes_keys.decode);
 611                         break;
 612                 default:
 613                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 614                         return -EINVAL;
 615                 }
 616
 617                 /* CCM digests must be between 4 and 16 and an even number */
 618                 if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
 619                         sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
 620                         (sess->auth.req_digest_len & 1) == 1) {
 621                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 622                         return -EINVAL;
 623                 }
 624                 break;
 625
 626         case RTE_CRYPTO_AEAD_AES_GCM:
 627                 sess->cipher.mode = IMB_CIPHER_GCM;
 628                 sess->auth.algo = IMB_AUTH_AES_GMAC;
 629
 630                 switch (xform->aead.key.length) {
 631                 case IMB_KEY_128_BYTES:
 632                         sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
 633                         IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
 634                                 &sess->cipher.gcm_key);
 635                         break;
 636                 case IMB_KEY_192_BYTES:
 637                         sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
 638                         IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
 639                                 &sess->cipher.gcm_key);
 640                         break;
 641                 case IMB_KEY_256_BYTES:
 642                         sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
 643                         IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
 644                                 &sess->cipher.gcm_key);
 645                         break;
 646                 default:
 647                         IPSEC_MB_LOG(ERR, "Invalid cipher key length");
 648                         return -EINVAL;
 649                 }
 650
 651                 /* GCM digest size must be between 1 and 16 */
 652                 if (sess->auth.req_digest_len == 0 ||
 653                                 sess->auth.req_digest_len > 16) {
 654                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 655                         return -EINVAL;
 656                 }
 657                 break;
 658
 659         case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
 660                 sess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305;
 661                 sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;
 662
 663                 if (xform->aead.key.length != 32) {
 664                         IPSEC_MB_LOG(ERR, "Invalid key length");
 665                         return -EINVAL;
 666                 }
 667                 sess->cipher.key_length_in_bytes = 32;
 668                 memcpy(sess->cipher.expanded_aes_keys.encode,
 669                         xform->aead.key.data, 32);
 670                 if (sess->auth.req_digest_len != 16) {
 671                         IPSEC_MB_LOG(ERR, "Invalid digest size\n");
 672                         return -EINVAL;
 673                 }
 674                 break;
 675         default:
 676                 IPSEC_MB_LOG(ERR, "Unsupported aead mode parameter");
 677                 return -ENOTSUP;
 678         }
 679
 680         return 0;
 681 }
 682
 683 /** Configure a aesni multi-buffer session from a crypto xform chain */
 684 static int
 685 aesni_mb_session_configure(IMB_MGR *mb_mgr,
 686                 void *priv_sess,
 687                 const struct rte_crypto_sym_xform *xform)
 688 {
 689         const struct rte_crypto_sym_xform *auth_xform = NULL;
 690         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 691         const struct rte_crypto_sym_xform *aead_xform = NULL;
 692         enum ipsec_mb_operation mode;
 693         struct aesni_mb_session *sess = (struct aesni_mb_session *) priv_sess;
 694         int ret;
 695
 696         ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
 697                                 &cipher_xform, &aead_xform);
 698         if (ret)
 699                 return ret;
 700
 701         /* Select Crypto operation - hash then cipher / cipher then hash */
 702         switch (mode) {
 703         case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
 704                 sess->chain_order = IMB_ORDER_HASH_CIPHER;
 705                 break;
 706         case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
 707         case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
 708                 sess->chain_order = IMB_ORDER_CIPHER_HASH;
 709                 break;
 710         case IPSEC_MB_OP_HASH_GEN_ONLY:
 711         case IPSEC_MB_OP_HASH_VERIFY_ONLY:
 712         case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
 713                 sess->chain_order = IMB_ORDER_HASH_CIPHER;
 714                 break;
 715         /*
 716          * Multi buffer library operates only at two modes,
 717          * IMB_ORDER_CIPHER_HASH and IMB_ORDER_HASH_CIPHER.
 718          * When doing ciphering only, chain order depends
 719          * on cipher operation: encryption is always
 720          * the first operation and decryption the last one.
 721          */
 722         case IPSEC_MB_OP_ENCRYPT_ONLY:
 723                 sess->chain_order = IMB_ORDER_CIPHER_HASH;
 724                 break;
 725         case IPSEC_MB_OP_DECRYPT_ONLY:
 726                 sess->chain_order = IMB_ORDER_HASH_CIPHER;
 727                 break;
 728         case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
 729                 sess->chain_order = IMB_ORDER_CIPHER_HASH;
 730                 sess->aead.aad_len = xform->aead.aad_length;
 731                 break;
 732         case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
 733                 sess->chain_order = IMB_ORDER_HASH_CIPHER;
 734                 sess->aead.aad_len = xform->aead.aad_length;
 735                 break;
 736         case IPSEC_MB_OP_NOT_SUPPORTED:
 737         default:
 738                 IPSEC_MB_LOG(ERR,
 739                         "Unsupported operation chain order parameter");
 740                 return -ENOTSUP;
 741         }
 742
 743         /* Default IV length = 0 */
 744         sess->iv.length = 0;
 745         sess->auth_iv.length = 0;
 746
 747         ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
 748         if (ret != 0) {
 749                 IPSEC_MB_LOG(ERR,
 750                         "Invalid/unsupported authentication parameters");
 751                 return ret;
 752         }
 753
 754         ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
 755                         cipher_xform);
 756         if (ret != 0) {
 757                 IPSEC_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 758                 return ret;
 759         }
 760
 761         if (aead_xform) {
 762                 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
 763                                 aead_xform);
 764                 if (ret != 0) {
 765                         IPSEC_MB_LOG(ERR,
 766                                 "Invalid/unsupported aead parameters");
 767                         return ret;
 768                 }
 769         }
 770
 771         return 0;
 772 }
 773
 774 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 775 /** Check DOCSIS security session configuration is valid */
 776 static int
 777 check_docsis_sec_session(struct rte_security_session_conf *conf)
 778 {
 779         struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;
 780         struct rte_security_docsis_xform *docsis = &conf->docsis;
 781
 782         /* Downlink: CRC generate -> Cipher encrypt */
 783         if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
 784
 785                 if (crypto_sym != NULL &&
 786                     crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 787                     crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
 788                     crypto_sym->cipher.algo ==
 789                                         RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
 790                     (crypto_sym->cipher.key.length == IMB_KEY_128_BYTES ||
 791                      crypto_sym->cipher.key.length == IMB_KEY_256_BYTES) &&
 792                     crypto_sym->cipher.iv.length == IMB_AES_BLOCK_SIZE &&
 793                     crypto_sym->next == NULL) {
 794                         return 0;
 795                 }
 796         /* Uplink: Cipher decrypt -> CRC verify */
 797         } else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {
 798
 799                 if (crypto_sym != NULL &&
 800                     crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 801                     crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
 802                     crypto_sym->cipher.algo ==
 803                                         RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
 804                     (crypto_sym->cipher.key.length == IMB_KEY_128_BYTES ||
 805                      crypto_sym->cipher.key.length == IMB_KEY_256_BYTES) &&
 806                     crypto_sym->cipher.iv.length == IMB_AES_BLOCK_SIZE &&
 807                     crypto_sym->next == NULL) {
 808                         return 0;
 809                 }
 810         }
 811
 812         return -EINVAL;
 813 }
 814
 815 /** Set DOCSIS security session auth (CRC) parameters */
 816 static int
 817 aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
 818                 struct rte_security_docsis_xform *xform)
 819 {
 820         if (xform == NULL) {
 821                 IPSEC_MB_LOG(ERR, "Invalid DOCSIS xform");
 822                 return -EINVAL;
 823         }
 824
 825         /* Select CRC generate/verify */
 826         if (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) {
 827                 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
 828                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 829         } else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
 830                 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
 831                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 832         } else {
 833                 IPSEC_MB_LOG(ERR, "Unsupported DOCSIS direction");
 834                 return -ENOTSUP;
 835         }
 836
 837         sess->auth.req_digest_len = RTE_ETHER_CRC_LEN;
 838         sess->auth.gen_digest_len = RTE_ETHER_CRC_LEN;
 839
 840         return 0;
 841 }
 842
 843 /**
 844  * Parse DOCSIS security session configuration and set private session
 845  * parameters
 846  */
 847 static int
 848 aesni_mb_set_docsis_sec_session_parameters(
 849                 __rte_unused struct rte_cryptodev *dev,
 850                 struct rte_security_session_conf *conf,
 851                 void *sess)
 852 {
 853         IMB_MGR  *mb_mgr = alloc_init_mb_mgr();
 854         struct rte_security_docsis_xform *docsis_xform;
 855         struct rte_crypto_sym_xform *cipher_xform;
 856         struct aesni_mb_session *ipsec_sess = sess;
 857         int ret = 0;
 858
 859         if (!mb_mgr)
 860                 return -ENOMEM;
 861
 862         ret = check_docsis_sec_session(conf);
 863         if (ret) {
 864                 IPSEC_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
 865                 goto error_exit;
 866         }
 867
 868         switch (conf->docsis.direction) {
 869         case RTE_SECURITY_DOCSIS_UPLINK:
 870                 ipsec_sess->chain_order = IMB_ORDER_CIPHER_HASH;
 871                 docsis_xform = &conf->docsis;
 872                 cipher_xform = conf->crypto_xform;
 873                 break;
 874         case RTE_SECURITY_DOCSIS_DOWNLINK:
 875                 ipsec_sess->chain_order = IMB_ORDER_HASH_CIPHER;
 876                 cipher_xform = conf->crypto_xform;
 877                 docsis_xform = &conf->docsis;
 878                 break;
 879         default:
 880                 IPSEC_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
 881                 ret = -EINVAL;
 882                 goto error_exit;
 883         }
 884
 885         /* Default IV length = 0 */
 886         ipsec_sess->iv.length = 0;
 887
 888         ret = aesni_mb_set_docsis_sec_session_auth_parameters(ipsec_sess,
 889                         docsis_xform);
 890         if (ret != 0) {
 891                 IPSEC_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
 892                 goto error_exit;
 893         }
 894
 895         ret = aesni_mb_set_session_cipher_parameters(mb_mgr,
 896                         ipsec_sess, cipher_xform);
 897
 898         if (ret != 0) {
 899                 IPSEC_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 900                 goto error_exit;
 901         }
 902
 903 error_exit:
 904         free_mb_mgr(mb_mgr);
 905         return ret;
 906 }
 907 #endif
 908
 909 static inline uint64_t
 910 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 911                 uint32_t oop)
 912 {
 913         struct rte_mbuf *m_src, *m_dst;
 914         uint8_t *p_src, *p_dst;
 915         uintptr_t u_src, u_dst;
 916         uint32_t cipher_end, auth_end;
 917
 918         /* Only cipher then hash needs special calculation. */
 919         if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH)
 920                 return op->sym->auth.data.offset;
 921
 922         m_src = op->sym->m_src;
 923         m_dst = op->sym->m_dst;
 924
 925         p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
 926         p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
 927         u_src = (uintptr_t)p_src;
 928         u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
 929
 930         /**
 931          * Copy the content between cipher offset and auth offset for generating
 932          * correct digest.
 933          */
 934         if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
 935                 memcpy(p_dst + op->sym->auth.data.offset,
 936                                 p_src + op->sym->auth.data.offset,
 937                                 op->sym->cipher.data.offset -
 938                                 op->sym->auth.data.offset);
 939
 940         /**
 941          * Copy the content between (cipher offset + length) and (auth offset +
 942          * length) for generating correct digest
 943          */
 944         cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
 945         auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
 946         if (cipher_end < auth_end)
 947                 memcpy(p_dst + cipher_end, p_src + cipher_end,
 948                                 auth_end - cipher_end);
 949
 950         /**
 951          * Since intel-ipsec-mb only supports positive values,
 952          * we need to deduct the correct offset between src and dst.
 953          */
 954
 955         return u_src < u_dst ? (u_dst - u_src) :
 956                         (UINT64_MAX - u_src + u_dst + 1);
 957 }
 958
 959 static inline void
 960 set_cpu_mb_job_params(IMB_JOB *job, struct aesni_mb_session *session,
 961                 union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
 962                 struct rte_crypto_va_iova_ptr *iv,
 963                 struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)
 964 {
 965         /* Set crypto operation */
 966         job->chain_order = session->chain_order;
 967
 968         /* Set cipher parameters */
 969         job->cipher_direction = session->cipher.direction;
 970         job->cipher_mode = session->cipher.mode;
 971
 972         job->key_len_in_bytes = session->cipher.key_length_in_bytes;
 973
 974         /* Set authentication parameters */
 975         job->hash_alg = session->auth.algo;
 976         job->iv = iv->va;
 977
 978         switch (job->hash_alg) {
 979         case IMB_AUTH_AES_XCBC:
 980                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
 981                 job->u.XCBC._k2 = session->auth.xcbc.k2;
 982                 job->u.XCBC._k3 = session->auth.xcbc.k3;
 983
 984                 job->enc_keys = session->cipher.expanded_aes_keys.encode;
 985                 job->dec_keys = session->cipher.expanded_aes_keys.decode;
 986                 break;
 987
 988         case IMB_AUTH_AES_CCM:
 989                 job->u.CCM.aad = (uint8_t *)aad->va + 18;
 990                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
 991                 job->enc_keys = session->cipher.expanded_aes_keys.encode;
 992                 job->dec_keys = session->cipher.expanded_aes_keys.decode;
 993                 job->iv++;
 994                 break;
 995
 996         case IMB_AUTH_AES_CMAC:
 997                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
 998                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
 999                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1000                 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1001                 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1002                 break;
1003
1004         case IMB_AUTH_AES_GMAC:
1005                 if (session->cipher.mode == IMB_CIPHER_GCM) {
1006                         job->u.GCM.aad = aad->va;
1007                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1008                 } else {
1009                         /* For GMAC */
1010                         job->u.GCM.aad = buf;
1011                         job->u.GCM.aad_len_in_bytes = len;
1012                         job->cipher_mode = IMB_CIPHER_GCM;
1013                 }
1014                 job->enc_keys = &session->cipher.gcm_key;
1015                 job->dec_keys = &session->cipher.gcm_key;
1016                 break;
1017
1018         case IMB_AUTH_CHACHA20_POLY1305:
1019                 job->u.CHACHA20_POLY1305.aad = aad->va;
1020                 job->u.CHACHA20_POLY1305.aad_len_in_bytes =
1021                         session->aead.aad_len;
1022                 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1023                 job->dec_keys = session->cipher.expanded_aes_keys.encode;
1024                 break;
1025         default:
1026                 job->u.HMAC._hashed_auth_key_xor_ipad =
1027                                 session->auth.pads.inner;
1028                 job->u.HMAC._hashed_auth_key_xor_opad =
1029                                 session->auth.pads.outer;
1030
1031                 if (job->cipher_mode == IMB_CIPHER_DES3) {
1032                         job->enc_keys = session->cipher.exp_3des_keys.ks_ptr;
1033                         job->dec_keys = session->cipher.exp_3des_keys.ks_ptr;
1034                 } else {
1035                         job->enc_keys = session->cipher.expanded_aes_keys.encode;
1036                         job->dec_keys = session->cipher.expanded_aes_keys.decode;
1037                 }
1038         }
1039
1040         /*
1041          * Multi-buffer library current only support returning a truncated
1042          * digest length as specified in the relevant IPsec RFCs
1043          */
1044
1045         /* Set digest location and length */
1046         job->auth_tag_output = digest;
1047         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1048
1049         /* Set IV parameters */
1050         job->iv_len_in_bytes = session->iv.length;
1051
1052         /* Data Parameters */
1053         job->src = buf;
1054         job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;
1055         job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;
1056         job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;
1057         if (job->hash_alg == IMB_AUTH_AES_GMAC &&
1058                         session->cipher.mode != IMB_CIPHER_GCM) {
1059                 job->msg_len_to_hash_in_bytes = 0;
1060                 job->msg_len_to_cipher_in_bytes = 0;
1061         } else {
1062                 job->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -
1063                         sofs.ofs.auth.tail;
1064                 job->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -
1065                         sofs.ofs.cipher.tail;
1066         }
1067
1068         job->user_data = udata;
1069 }
1070
1071 /**
1072  * Process a crypto operation and complete a IMB_JOB job structure for
1073  * submission to the multi buffer library for processing.
1074  *
1075  * @param       qp              queue pair
1076  * @param       job             IMB_JOB structure to fill
1077  * @param       op              crypto op to process
1078  * @param       digest_idx      ID for digest to use
1079  *
1080  * @return
1081  * - 0 on success, the IMB_JOB will be filled
1082  * - -1 if invalid session, IMB_JOB will not be filled
1083  */
1084 static inline int
1085 set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
1086                 struct rte_crypto_op *op, uint8_t *digest_idx)
1087 {
1088         struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
1089         struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
1090         struct aesni_mb_session *session;
1091         uint32_t m_offset, oop;
1092
1093         session = ipsec_mb_get_session_private(qp, op);
1094         if (session == NULL) {
1095                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1096                 return -1;
1097         }
1098
1099         /* Set crypto operation */
1100         job->chain_order = session->chain_order;
1101
1102         /* Set cipher parameters */
1103         job->cipher_direction = session->cipher.direction;
1104         job->cipher_mode = session->cipher.mode;
1105
1106         job->key_len_in_bytes = session->cipher.key_length_in_bytes;
1107
1108         /* Set authentication parameters */
1109         job->hash_alg = session->auth.algo;
1110
1111         const int aead = is_aead_algo(job->hash_alg, job->cipher_mode);
1112
1113         switch (job->hash_alg) {
1114         case IMB_AUTH_AES_XCBC:
1115                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
1116                 job->u.XCBC._k2 = session->auth.xcbc.k2;
1117                 job->u.XCBC._k3 = session->auth.xcbc.k3;
1118
1119                 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1120                 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1121                 break;
1122
1123         case IMB_AUTH_AES_CCM:
1124                 job->u.CCM.aad = op->sym->aead.aad.data + 18;
1125                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1126                 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1127                 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1128                 break;
1129
1130         case IMB_AUTH_AES_CMAC:
1131                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1132                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1133                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1134                 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1135                 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1136                 break;
1137
1138         case IMB_AUTH_AES_GMAC:
1139                 if (session->cipher.mode == IMB_CIPHER_GCM) {
1140                         job->u.GCM.aad = op->sym->aead.aad.data;
1141                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1142                 } else {
1143                         /* For GMAC */
1144                         job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
1145                                         uint8_t *, op->sym->auth.data.offset);
1146                         job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
1147                         job->cipher_mode = IMB_CIPHER_GCM;
1148                 }
1149                 job->enc_keys = &session->cipher.gcm_key;
1150                 job->dec_keys = &session->cipher.gcm_key;
1151                 break;
1152         case IMB_AUTH_ZUC_EIA3_BITLEN:
1153                 job->u.ZUC_EIA3._key = session->auth.zuc_auth_key;
1154                 job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1155                                                 session->auth_iv.offset);
1156                 break;
1157         case IMB_AUTH_SNOW3G_UIA2_BITLEN:
1158                 job->u.SNOW3G_UIA2._key = (void *)
1159                         &session->auth.pKeySched_snow3g_auth;
1160                 job->u.SNOW3G_UIA2._iv =
1161                         rte_crypto_op_ctod_offset(op, uint8_t *,
1162                                                 session->auth_iv.offset);
1163                 break;
1164         case IMB_AUTH_KASUMI_UIA1:
1165                 job->u.KASUMI_UIA1._key = (void *)
1166                         &session->auth.pKeySched_kasumi_auth;
1167                 break;
1168         case IMB_AUTH_CHACHA20_POLY1305:
1169                 job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
1170                 job->u.CHACHA20_POLY1305.aad_len_in_bytes =
1171                         session->aead.aad_len;
1172                 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1173                 job->dec_keys = session->cipher.expanded_aes_keys.encode;
1174                 break;
1175         default:
1176                 job->u.HMAC._hashed_auth_key_xor_ipad =
1177                         session->auth.pads.inner;
1178                 job->u.HMAC._hashed_auth_key_xor_opad =
1179                         session->auth.pads.outer;
1180
1181                 if (job->cipher_mode == IMB_CIPHER_DES3) {
1182                         job->enc_keys = session->cipher.exp_3des_keys.ks_ptr;
1183                         job->dec_keys = session->cipher.exp_3des_keys.ks_ptr;
1184                 } else {
1185                         job->enc_keys = session->cipher.expanded_aes_keys.encode;
1186                         job->dec_keys = session->cipher.expanded_aes_keys.decode;
1187                 }
1188         }
1189
1190         if (aead)
1191                 m_offset = op->sym->aead.data.offset;
1192         else
1193                 m_offset = op->sym->cipher.data.offset;
1194
1195         if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
1196                 job->enc_keys = session->cipher.zuc_cipher_key;
1197                 job->dec_keys = session->cipher.zuc_cipher_key;
1198         } else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
1199                 job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
1200                 m_offset = 0;
1201         } else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) {
1202                 job->enc_keys = &session->cipher.pKeySched_kasumi_cipher;
1203                 m_offset = 0;
1204         }
1205
1206         if (!op->sym->m_dst) {
1207                 /* in-place operation */
1208                 m_dst = m_src;
1209                 oop = 0;
1210         } else if (op->sym->m_dst == op->sym->m_src) {
1211                 /* in-place operation */
1212                 m_dst = m_src;
1213                 oop = 0;
1214         } else {
1215                 /* out-of-place operation */
1216                 m_dst = op->sym->m_dst;
1217                 oop = 1;
1218         }
1219
1220         /* Set digest output location */
1221         if (job->hash_alg != IMB_AUTH_NULL &&
1222                         session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1223                 job->auth_tag_output = qp_data->temp_digests[*digest_idx];
1224                 *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
1225         } else {
1226                 if (aead)
1227                         job->auth_tag_output = op->sym->aead.digest.data;
1228                 else
1229                         job->auth_tag_output = op->sym->auth.digest.data;
1230
1231                 if (session->auth.req_digest_len !=
1232                                 session->auth.gen_digest_len) {
1233                         job->auth_tag_output =
1234                                 qp_data->temp_digests[*digest_idx];
1235                         *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
1236                 }
1237         }
1238         /*
1239          * Multi-buffer library current only support returning a truncated
1240          * digest length as specified in the relevant IPsec RFCs
1241          */
1242
1243         /* Set digest length */
1244         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1245
1246         /* Set IV parameters */
1247         job->iv_len_in_bytes = session->iv.length;
1248
1249         /* Data Parameters */
1250         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1251         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
1252
1253         switch (job->hash_alg) {
1254         case IMB_AUTH_AES_CCM:
1255                 job->cipher_start_src_offset_in_bytes =
1256                                 op->sym->aead.data.offset;
1257                 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
1258                 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
1259                 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
1260
1261                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1262                         session->iv.offset + 1);
1263                 break;
1264
1265         case IMB_AUTH_AES_GMAC:
1266                 if (session->cipher.mode == IMB_CIPHER_GCM) {
1267                         job->cipher_start_src_offset_in_bytes =
1268                                         op->sym->aead.data.offset;
1269                         job->hash_start_src_offset_in_bytes =
1270                                         op->sym->aead.data.offset;
1271                         job->msg_len_to_cipher_in_bytes =
1272                                         op->sym->aead.data.length;
1273                         job->msg_len_to_hash_in_bytes =
1274                                         op->sym->aead.data.length;
1275                 } else {
1276                         job->cipher_start_src_offset_in_bytes =
1277                                         op->sym->auth.data.offset;
1278                         job->hash_start_src_offset_in_bytes =
1279                                         op->sym->auth.data.offset;
1280                         job->msg_len_to_cipher_in_bytes = 0;
1281                         job->msg_len_to_hash_in_bytes = 0;
1282                 }
1283
1284                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1285                                 session->iv.offset);
1286                 break;
1287
1288         case IMB_AUTH_CHACHA20_POLY1305:
1289                 job->cipher_start_src_offset_in_bytes =
1290                         op->sym->aead.data.offset;
1291                 job->hash_start_src_offset_in_bytes =
1292                         op->sym->aead.data.offset;
1293                 job->msg_len_to_cipher_in_bytes =
1294                                 op->sym->aead.data.length;
1295                 job->msg_len_to_hash_in_bytes =
1296                                         op->sym->aead.data.length;
1297
1298                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1299                                 session->iv.offset);
1300                 break;
1301         default:
1302                 /* For SNOW3G, length and offsets are already in bits */
1303                 job->cipher_start_src_offset_in_bytes =
1304                                 op->sym->cipher.data.offset;
1305                 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
1306
1307                 job->hash_start_src_offset_in_bytes = auth_start_offset(op,
1308                                 session, oop);
1309                 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
1310
1311                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1312                         session->iv.offset);
1313         }
1314
1315         if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
1316                 job->msg_len_to_cipher_in_bytes >>= 3;
1317         else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)
1318                 job->msg_len_to_hash_in_bytes >>= 3;
1319
1320         /* Set user data to be crypto operation data struct */
1321         job->user_data = op;
1322
1323         return 0;
1324 }
1325
1326 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1327 /**
1328  * Process a crypto operation containing a security op and complete a
1329  * IMB_JOB job structure for submission to the multi buffer library for
1330  * processing.
1331  */
1332 static inline int
1333 set_sec_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
1334                         struct rte_crypto_op *op, uint8_t *digest_idx)
1335 {
1336         struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
1337         struct rte_mbuf *m_src, *m_dst;
1338         struct rte_crypto_sym_op *sym;
1339         struct aesni_mb_session *session = NULL;
1340
1341         if (unlikely(op->sess_type != RTE_CRYPTO_OP_SECURITY_SESSION)) {
1342                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1343                 return -1;
1344         }
1345         session = (struct aesni_mb_session *)
1346                 get_sec_session_private_data(op->sym->sec_session);
1347
1348         if (unlikely(session == NULL)) {
1349                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1350                 return -1;
1351         }
1352         /* Only DOCSIS protocol operations supported now */
1353         if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||
1354                         session->auth.algo != IMB_AUTH_DOCSIS_CRC32) {
1355                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1356                 return -1;
1357         }
1358
1359         sym = op->sym;
1360         m_src = sym->m_src;
1361
1362         if (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {
1363                 /* in-place operation */
1364                 m_dst = m_src;
1365         } else {
1366                 /* out-of-place operation not supported */
1367                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1368                 return -ENOTSUP;
1369         }
1370
1371         /* Set crypto operation */
1372         job->chain_order = session->chain_order;
1373
1374         /* Set cipher parameters */
1375         job->cipher_direction = session->cipher.direction;
1376         job->cipher_mode = session->cipher.mode;
1377
1378         job->key_len_in_bytes = session->cipher.key_length_in_bytes;
1379         job->enc_keys = session->cipher.expanded_aes_keys.encode;
1380         job->dec_keys = session->cipher.expanded_aes_keys.decode;
1381
1382         /* Set IV parameters */
1383         job->iv_len_in_bytes = session->iv.length;
1384         job->iv = (uint8_t *)op + session->iv.offset;
1385
1386         /* Set authentication parameters */
1387         job->hash_alg = session->auth.algo;
1388
1389         /* Set digest output location */
1390         job->auth_tag_output = qp_data->temp_digests[*digest_idx];
1391         *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
1392
1393         /* Set digest length */
1394         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1395
1396         /* Set data parameters */
1397         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1398         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,
1399                                                 sym->cipher.data.offset);
1400
1401         job->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;
1402         job->msg_len_to_cipher_in_bytes = sym->cipher.data.length;
1403
1404         job->hash_start_src_offset_in_bytes = sym->auth.data.offset;
1405         job->msg_len_to_hash_in_bytes = sym->auth.data.length;
1406
1407         job->user_data = op;
1408
1409         return 0;
1410 }
1411
1412 static inline void
1413 verify_docsis_sec_crc(IMB_JOB *job, uint8_t *status)
1414 {
1415         uint16_t crc_offset;
1416         uint8_t *crc;
1417
1418         if (!job->msg_len_to_hash_in_bytes)
1419                 return;
1420
1421         crc_offset = job->hash_start_src_offset_in_bytes +
1422                         job->msg_len_to_hash_in_bytes -
1423                         job->cipher_start_src_offset_in_bytes;
1424         crc = job->dst + crc_offset;
1425
1426         /* Verify CRC (at the end of the message) */
1427         if (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0)
1428                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1429 }
1430 #endif
1431
1432 static inline void
1433 verify_digest(IMB_JOB *job, void *digest, uint16_t len, uint8_t *status)
1434 {
1435         /* Verify digest if required */
1436         if (memcmp(job->auth_tag_output, digest, len) != 0)
1437                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1438 }
1439
1440 static inline void
1441 generate_digest(IMB_JOB *job, struct rte_crypto_op *op,
1442                 struct aesni_mb_session *sess)
1443 {
1444         /* No extra copy needed */
1445         if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
1446                 return;
1447
1448         /*
1449          * This can only happen for HMAC, so only digest
1450          * for authentication algos is required
1451          */
1452         memcpy(op->sym->auth.digest.data, job->auth_tag_output,
1453                         sess->auth.req_digest_len);
1454 }
1455
1456 /**
1457  * Process a completed job and return rte_mbuf which job processed
1458  *
1459  * @param qp    Queue Pair to process
1460  * @param job   IMB_JOB job to process
1461  *
1462  * @return
1463  * - Returns processed crypto operation.
1464  * - Returns NULL on invalid job
1465  */
1466 static inline struct rte_crypto_op *
1467 post_process_mb_job(struct ipsec_mb_qp *qp, IMB_JOB *job)
1468 {
1469         struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
1470         struct aesni_mb_session *sess = NULL;
1471         uint32_t driver_id = ipsec_mb_get_driver_id(
1472                                                 IPSEC_MB_PMD_TYPE_AESNI_MB);
1473
1474 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1475         uint8_t is_docsis_sec = 0;
1476
1477         if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1478                 /*
1479                  * Assuming at this point that if it's a security type op, that
1480                  * this is for DOCSIS
1481                  */
1482                 is_docsis_sec = 1;
1483                 sess = get_sec_session_private_data(op->sym->sec_session);
1484         } else
1485 #endif
1486         {
1487                 sess = get_sym_session_private_data(op->sym->session,
1488                                                 driver_id);
1489         }
1490
1491         if (unlikely(sess == NULL)) {
1492                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1493                 return op;
1494         }
1495
1496         if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
1497                 switch (job->status) {
1498                 case IMB_STATUS_COMPLETED:
1499                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1500
1501                         if (job->hash_alg == IMB_AUTH_NULL)
1502                                 break;
1503
1504                         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1505                                 if (is_aead_algo(job->hash_alg,
1506                                                 sess->cipher.mode))
1507                                         verify_digest(job,
1508                                                 op->sym->aead.digest.data,
1509                                                 sess->auth.req_digest_len,
1510                                                 &op->status);
1511 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1512                                 else if (is_docsis_sec)
1513                                         verify_docsis_sec_crc(job,
1514                                                 &op->status);
1515 #endif
1516                                 else
1517                                         verify_digest(job,
1518                                                 op->sym->auth.digest.data,
1519                                                 sess->auth.req_digest_len,
1520                                                 &op->status);
1521                         } else
1522                                 generate_digest(job, op, sess);
1523                         break;
1524                 default:
1525                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1526                 }
1527         }
1528
1529         /* Free session if a session-less crypto op */
1530         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1531                 memset(sess, 0, sizeof(struct aesni_mb_session));
1532                 memset(op->sym->session, 0,
1533                         rte_cryptodev_sym_get_existing_header_session_size(
1534                                 op->sym->session));
1535                 rte_mempool_put(qp->sess_mp_priv, sess);
1536                 rte_mempool_put(qp->sess_mp, op->sym->session);
1537                 op->sym->session = NULL;
1538         }
1539
1540         return op;
1541 }
1542
1543 static inline void
1544 post_process_mb_sync_job(IMB_JOB *job)
1545 {
1546         uint32_t *st;
1547
1548         st = job->user_data;
1549         st[0] = (job->status == IMB_STATUS_COMPLETED) ? 0 : EBADMSG;
1550 }
1551
1552 /**
1553  * Process a completed IMB_JOB job and keep processing jobs until
1554  * get_completed_job return NULL
1555  *
1556  * @param qp            Queue Pair to process
1557  * @param mb_mgr        IMB_MGR to use
1558  * @param job           IMB_JOB job
1559  * @param ops           crypto ops to fill
1560  * @param nb_ops        number of crypto ops
1561  *
1562  * @return
1563  * - Number of processed jobs
1564  */
1565 static unsigned
1566 handle_completed_jobs(struct ipsec_mb_qp *qp, IMB_MGR *mb_mgr,
1567                 IMB_JOB *job, struct rte_crypto_op **ops,
1568                 uint16_t nb_ops)
1569 {
1570         struct rte_crypto_op *op = NULL;
1571         uint16_t processed_jobs = 0;
1572
1573         while (job != NULL) {
1574                 op = post_process_mb_job(qp, job);
1575
1576                 if (op) {
1577                         ops[processed_jobs++] = op;
1578                         qp->stats.dequeued_count++;
1579                 } else {
1580                         qp->stats.dequeue_err_count++;
1581                         break;
1582                 }
1583                 if (processed_jobs == nb_ops)
1584                         break;
1585
1586                 job = IMB_GET_COMPLETED_JOB(mb_mgr);
1587         }
1588
1589         return processed_jobs;
1590 }
1591
1592 static inline uint32_t
1593 handle_completed_sync_jobs(IMB_JOB *job, IMB_MGR *mb_mgr)
1594 {
1595         uint32_t i;
1596
1597         for (i = 0; job != NULL; i++, job = IMB_GET_COMPLETED_JOB(mb_mgr))
1598                 post_process_mb_sync_job(job);
1599
1600         return i;
1601 }
1602
1603 static inline uint32_t
1604 flush_mb_sync_mgr(IMB_MGR *mb_mgr)
1605 {
1606         IMB_JOB *job;
1607
1608         job = IMB_FLUSH_JOB(mb_mgr);
1609         return handle_completed_sync_jobs(job, mb_mgr);
1610 }
1611
1612 static inline uint16_t
1613 flush_mb_mgr(struct ipsec_mb_qp *qp, IMB_MGR *mb_mgr,
1614                 struct rte_crypto_op **ops, uint16_t nb_ops)
1615 {
1616         int processed_ops = 0;
1617
1618         /* Flush the remaining jobs */
1619         IMB_JOB *job = IMB_FLUSH_JOB(mb_mgr);
1620
1621         if (job)
1622                 processed_ops += handle_completed_jobs(qp, mb_mgr, job,
1623                                 &ops[processed_ops], nb_ops - processed_ops);
1624
1625         return processed_ops;
1626 }
1627
1628 static inline IMB_JOB *
1629 set_job_null_op(IMB_JOB *job, struct rte_crypto_op *op)
1630 {
1631         job->chain_order = IMB_ORDER_HASH_CIPHER;
1632         job->cipher_mode = IMB_CIPHER_NULL;
1633         job->hash_alg = IMB_AUTH_NULL;
1634         job->cipher_direction = IMB_DIR_DECRYPT;
1635
1636         /* Set user data to be crypto operation data struct */
1637         job->user_data = op;
1638
1639         return job;
1640 }
1641
1642 static uint16_t
1643 aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1644                 uint16_t nb_ops)
1645 {
1646         struct ipsec_mb_qp *qp = queue_pair;
1647         IMB_MGR *mb_mgr = qp->mb_mgr;
1648         struct rte_crypto_op *op;
1649         IMB_JOB *job;
1650         int retval, processed_jobs = 0;
1651
1652         if (unlikely(nb_ops == 0 || mb_mgr == NULL))
1653                 return 0;
1654
1655         uint8_t digest_idx = qp->digest_idx;
1656
1657         do {
1658                 /* Get next free mb job struct from mb manager */
1659                 job = IMB_GET_NEXT_JOB(mb_mgr);
1660                 if (unlikely(job == NULL)) {
1661                         /* if no free mb job structs we need to flush mb_mgr */
1662                         processed_jobs += flush_mb_mgr(qp, mb_mgr,
1663                                         &ops[processed_jobs],
1664                                         nb_ops - processed_jobs);
1665
1666                         if (nb_ops == processed_jobs)
1667                                 break;
1668
1669                         job = IMB_GET_NEXT_JOB(mb_mgr);
1670                 }
1671
1672                 /*
1673                  * Get next operation to process from ingress queue.
1674                  * There is no need to return the job to the IMB_MGR
1675                  * if there are no more operations to process, since the IMB_MGR
1676                  * can use that pointer again in next get_next calls.
1677                  */
1678                 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1679                 if (retval < 0)
1680                         break;
1681
1682 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1683                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1684                         retval = set_sec_mb_job_params(job, qp, op,
1685                                                 &digest_idx);
1686                 else
1687 #endif
1688                         retval = set_mb_job_params(job, qp, op,
1689                                 &digest_idx);
1690
1691                 if (unlikely(retval != 0)) {
1692                         qp->stats.dequeue_err_count++;
1693                         set_job_null_op(job, op);
1694                 }
1695
1696                 /* Submit job to multi-buffer for processing */
1697 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1698                 job = IMB_SUBMIT_JOB(mb_mgr);
1699 #else
1700                 job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
1701 #endif
1702                 /*
1703                  * If submit returns a processed job then handle it,
1704                  * before submitting subsequent jobs
1705                  */
1706                 if (job)
1707                         processed_jobs += handle_completed_jobs(qp, mb_mgr,
1708                                         job, &ops[processed_jobs],
1709                                         nb_ops - processed_jobs);
1710
1711         } while (processed_jobs < nb_ops);
1712
1713         qp->digest_idx = digest_idx;
1714
1715         if (processed_jobs < 1)
1716                 processed_jobs += flush_mb_mgr(qp, mb_mgr,
1717                                 &ops[processed_jobs],
1718                                 nb_ops - processed_jobs);
1719
1720         return processed_jobs;
1721 }
1722
1723
1724 static inline void
1725 ipsec_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
1726 {
1727         uint32_t i;
1728
1729         for (i = 0; i != vec->num; ++i)
1730                 vec->status[i] = err;
1731 }
1732
1733 static inline int
1734 check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)
1735 {
1736         /* no multi-seg support with current AESNI-MB PMD */
1737         if (sgl->num != 1)
1738                 return -ENOTSUP;
1739         else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)
1740                 return -EINVAL;
1741         return 0;
1742 }
1743
1744 static inline IMB_JOB *
1745 submit_sync_job(IMB_MGR *mb_mgr)
1746 {
1747 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1748         return IMB_SUBMIT_JOB(mb_mgr);
1749 #else
1750         return IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
1751 #endif
1752 }
1753
1754 static inline uint32_t
1755 generate_sync_dgst(struct rte_crypto_sym_vec *vec,
1756         const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1757 {
1758         uint32_t i, k;
1759
1760         for (i = 0, k = 0; i != vec->num; i++) {
1761                 if (vec->status[i] == 0) {
1762                         memcpy(vec->digest[i].va, dgst[i], len);
1763                         k++;
1764                 }
1765         }
1766
1767         return k;
1768 }
1769
1770 static inline uint32_t
1771 verify_sync_dgst(struct rte_crypto_sym_vec *vec,
1772         const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1773 {
1774         uint32_t i, k;
1775
1776         for (i = 0, k = 0; i != vec->num; i++) {
1777                 if (vec->status[i] == 0) {
1778                         if (memcmp(vec->digest[i].va, dgst[i], len) != 0)
1779                                 vec->status[i] = EBADMSG;
1780                         else
1781                                 k++;
1782                 }
1783         }
1784
1785         return k;
1786 }
1787
1788 static uint32_t
1789 aesni_mb_process_bulk(struct rte_cryptodev *dev,
1790         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
1791         struct rte_crypto_sym_vec *vec)
1792 {
1793         int32_t ret;
1794         uint32_t i, j, k, len;
1795         void *buf;
1796         IMB_JOB *job;
1797         IMB_MGR *mb_mgr;
1798         struct aesni_mb_session *s;
1799         uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];
1800
1801         s = get_sym_session_private_data(sess, dev->driver_id);
1802         if (s == NULL) {
1803                 ipsec_mb_fill_error_code(vec, EINVAL);
1804                 return 0;
1805         }
1806
1807         /* get per-thread MB MGR, create one if needed */
1808         mb_mgr = get_per_thread_mb_mgr();
1809         if (unlikely(mb_mgr == NULL))
1810                 return 0;
1811
1812         for (i = 0, j = 0, k = 0; i != vec->num; i++) {
1813                 ret = check_crypto_sgl(sofs, vec->src_sgl + i);
1814                 if (ret != 0) {
1815                         vec->status[i] = ret;
1816                         continue;
1817                 }
1818
1819                 buf = vec->src_sgl[i].vec[0].base;
1820                 len = vec->src_sgl[i].vec[0].len;
1821
1822                 job = IMB_GET_NEXT_JOB(mb_mgr);
1823                 if (job == NULL) {
1824                         k += flush_mb_sync_mgr(mb_mgr);
1825                         job = IMB_GET_NEXT_JOB(mb_mgr);
1826                         RTE_ASSERT(job != NULL);
1827                 }
1828
1829                 /* Submit job for processing */
1830                 set_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i],
1831                         &vec->aad[i], tmp_dgst[i], &vec->status[i]);
1832                 job = submit_sync_job(mb_mgr);
1833                 j++;
1834
1835                 /* handle completed jobs */
1836                 k += handle_completed_sync_jobs(job, mb_mgr);
1837         }
1838
1839         /* flush remaining jobs */
1840         while (k != j)
1841                 k += flush_mb_sync_mgr(mb_mgr);
1842
1843         /* finish processing for successful jobs: check/update digest */
1844         if (k != 0) {
1845                 if (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1846                         k = verify_sync_dgst(vec,
1847                                 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1848                                 s->auth.req_digest_len);
1849                 else
1850                         k = generate_sync_dgst(vec,
1851                                 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1852                                 s->auth.req_digest_len);
1853         }
1854
1855         return k;
1856 }
1857
1858 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
1859         .dev_configure = ipsec_mb_config,
1860         .dev_start = ipsec_mb_start,
1861         .dev_stop = ipsec_mb_stop,
1862         .dev_close = ipsec_mb_close,
1863
1864         .stats_get = ipsec_mb_stats_get,
1865         .stats_reset = ipsec_mb_stats_reset,
1866
1867         .dev_infos_get = ipsec_mb_info_get,
1868
1869         .queue_pair_setup = ipsec_mb_qp_setup,
1870         .queue_pair_release = ipsec_mb_qp_release,
1871
1872         .sym_cpu_process = aesni_mb_process_bulk,
1873
1874         .sym_session_get_size = ipsec_mb_sym_session_get_size,
1875         .sym_session_configure = ipsec_mb_sym_session_configure,
1876         .sym_session_clear = ipsec_mb_sym_session_clear
1877 };
1878
1879 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1880 /**
1881  * Configure a aesni multi-buffer session from a security session
1882  * configuration
1883  */
1884 static int
1885 aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
1886                 struct rte_security_session *sess,
1887                 struct rte_mempool *mempool)
1888 {
1889         void *sess_private_data;
1890         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1891         int ret;
1892
1893         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
1894                         conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
1895                 IPSEC_MB_LOG(ERR, "Invalid security protocol");
1896                 return -EINVAL;
1897         }
1898
1899         if (rte_mempool_get(mempool, &sess_private_data)) {
1900                 IPSEC_MB_LOG(ERR, "Couldn't get object from session mempool");
1901                 return -ENOMEM;
1902         }
1903
1904         ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
1905                         sess_private_data);
1906
1907         if (ret != 0) {
1908                 IPSEC_MB_LOG(ERR, "Failed to configure session parameters");
1909
1910                 /* Return session to mempool */
1911                 rte_mempool_put(mempool, sess_private_data);
1912                 return ret;
1913         }
1914
1915         set_sec_session_private_data(sess, sess_private_data);
1916
1917         return ret;
1918 }
1919
1920 /** Clear the memory of session so it does not leave key material behind */
1921 static int
1922 aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
1923                 struct rte_security_session *sess)
1924 {
1925         void *sess_priv = get_sec_session_private_data(sess);
1926
1927         if (sess_priv) {
1928                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1929
1930                 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
1931                 set_sec_session_private_data(sess, NULL);
1932                 rte_mempool_put(sess_mp, sess_priv);
1933         }
1934         return 0;
1935 }
1936
1937 /** Get security capabilities for aesni multi-buffer */
1938 static const struct rte_security_capability *
1939 aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
1940 {
1941         return aesni_mb_pmd_security_cap;
1942 }
1943
1944 static struct rte_security_ops aesni_mb_pmd_sec_ops = {
1945                 .session_create = aesni_mb_pmd_sec_sess_create,
1946                 .session_update = NULL,
1947                 .session_stats_get = NULL,
1948                 .session_destroy = aesni_mb_pmd_sec_sess_destroy,
1949                 .set_pkt_metadata = NULL,
1950                 .capabilities_get = aesni_mb_pmd_sec_capa_get
1951 };
1952
1953 struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
1954
1955 static int
1956 aesni_mb_configure_dev(struct rte_cryptodev *dev)
1957 {
1958         struct rte_security_ctx *security_instance;
1959
1960         security_instance = rte_malloc("aesni_mb_sec",
1961                                 sizeof(struct rte_security_ctx),
1962                                 RTE_CACHE_LINE_SIZE);
1963         if (security_instance != NULL) {
1964                 security_instance->device = (void *)dev;
1965                 security_instance->ops = rte_aesni_mb_pmd_sec_ops;
1966                 security_instance->sess_cnt = 0;
1967                 dev->security_ctx = security_instance;
1968
1969                 return 0;
1970         }
1971
1972         return -ENOMEM;
1973 }
1974
1975 #endif
1976
1977 static int
1978 aesni_mb_probe(struct rte_vdev_device *vdev)
1979 {
1980         return ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_AESNI_MB);
1981 }
1982
1983 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
1984         .probe = aesni_mb_probe,
1985         .remove = ipsec_mb_remove
1986 };
1987
1988 static struct cryptodev_driver aesni_mb_crypto_drv;
1989
1990 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD,
1991         cryptodev_aesni_mb_pmd_drv);
1992 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
1993 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
1994                         "max_nb_queue_pairs=<int> socket_id=<int>");
1995 RTE_PMD_REGISTER_CRYPTO_DRIVER(
1996         aesni_mb_crypto_drv,
1997         cryptodev_aesni_mb_pmd_drv.driver,
1998         pmd_driver_id_aesni_mb);
1999
2000 /* Constructor function to register aesni-mb PMD */
2001 RTE_INIT(ipsec_mb_register_aesni_mb)
2002 {
2003         struct ipsec_mb_internals *aesni_mb_data =
2004                 &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_AESNI_MB];
2005
2006         aesni_mb_data->caps = aesni_mb_capabilities;
2007         aesni_mb_data->dequeue_burst = aesni_mb_dequeue_burst;
2008         aesni_mb_data->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2009                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2010                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
2011                         RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
2012                         RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
2013                         RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
2014
2015         aesni_mb_data->internals_priv_size = 0;
2016         aesni_mb_data->ops = &aesni_mb_pmd_ops;
2017         aesni_mb_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);
2018         aesni_mb_data->queue_pair_configure = NULL;
2019 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
2020         aesni_mb_data->security_ops = &aesni_mb_pmd_sec_ops;
2021         aesni_mb_data->dev_config = aesni_mb_configure_dev;
2022         aesni_mb_data->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
2023 #endif
2024         aesni_mb_data->session_configure = aesni_mb_session_configure;
2025         aesni_mb_data->session_priv_size = sizeof(struct aesni_mb_session);
2026 }