drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <intel-ipsec-mb.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_hexdump.h>
   9 #include <rte_cryptodev.h>
  10 #include <rte_cryptodev_pmd.h>
  11 #include <rte_bus_vdev.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cpuflags.h>
  14 #include <rte_per_lcore.h>
  15 #include <rte_ether.h>
  16
  17 #include "aesni_mb_pmd_private.h"
  18
  19 #define AES_CCM_DIGEST_MIN_LEN 4
  20 #define AES_CCM_DIGEST_MAX_LEN 16
  21 #define HMAC_MAX_BLOCK_SIZE 128
  22 static uint8_t cryptodev_driver_id;
  23
  24 /*
  25  * Needed to support CPU-CRYPTO API (rte_cryptodev_sym_cpu_crypto_process),
  26  * as we still use JOB based API even for synchronous processing.
  27  */
  28 static RTE_DEFINE_PER_LCORE(MB_MGR *, sync_mb_mgr);
  29
  30 typedef void (*hash_one_block_t)(const void *data, void *digest);
  31 typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
  32
  33 /**
  34  * Calculate the authentication pre-computes
  35  *
  36  * @param one_block_hash        Function pointer to calculate digest on ipad/opad
  37  * @param ipad                  Inner pad output byte array
  38  * @param opad                  Outer pad output byte array
  39  * @param hkey                  Authentication key
  40  * @param hkey_len              Authentication key length
  41  * @param blocksize             Block size of selected hash algo
  42  */
  43 static void
  44 calculate_auth_precomputes(hash_one_block_t one_block_hash,
  45                 uint8_t *ipad, uint8_t *opad,
  46                 const uint8_t *hkey, uint16_t hkey_len,
  47                 uint16_t blocksize)
  48 {
  49         unsigned i, length;
  50
  51         uint8_t ipad_buf[blocksize] __rte_aligned(16);
  52         uint8_t opad_buf[blocksize] __rte_aligned(16);
  53
  54         /* Setup inner and outer pads */
  55         memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
  56         memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
  57
  58         /* XOR hash key with inner and outer pads */
  59         length = hkey_len > blocksize ? blocksize : hkey_len;
  60
  61         for (i = 0; i < length; i++) {
  62                 ipad_buf[i] ^= hkey[i];
  63                 opad_buf[i] ^= hkey[i];
  64         }
  65
  66         /* Compute partial hashes */
  67         (*one_block_hash)(ipad_buf, ipad);
  68         (*one_block_hash)(opad_buf, opad);
  69
  70         /* Clean up stack */
  71         memset(ipad_buf, 0, blocksize);
  72         memset(opad_buf, 0, blocksize);
  73 }
  74
  75 /** Get xform chain order */
  76 static enum aesni_mb_operation
  77 aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
  78 {
  79         if (xform == NULL)
  80                 return AESNI_MB_OP_NOT_SUPPORTED;
  81
  82         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  83                 if (xform->next == NULL)
  84                         return AESNI_MB_OP_CIPHER_ONLY;
  85                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  86                         return AESNI_MB_OP_CIPHER_HASH;
  87         }
  88
  89         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  90                 if (xform->next == NULL)
  91                         return AESNI_MB_OP_HASH_ONLY;
  92                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
  93                         return AESNI_MB_OP_HASH_CIPHER;
  94         }
  95 #if IMB_VERSION_NUM > IMB_VERSION(0, 52, 0)
  96         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  97                 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
  98                         /*
  99                          * CCM requires to hash first and cipher later
 100                          * when encrypting
 101                          */
 102                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 103                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 104                         else
 105                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 106                 } else {
 107                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 108                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 109                         else
 110                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 111                 }
 112         }
 113 #else
 114         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 115                 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
 116                                 xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
 117                         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 118                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 119                         else
 120                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 121                 }
 122         }
 123 #endif
 124
 125         return AESNI_MB_OP_NOT_SUPPORTED;
 126 }
 127
 128 static inline int
 129 is_aead_algo(JOB_HASH_ALG hash_alg, JOB_CIPHER_MODE cipher_mode)
 130 {
 131 #if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
 132         return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 || hash_alg == AES_CCM ||
 133                 (hash_alg == AES_GMAC && cipher_mode == GCM));
 134 #else
 135         return ((hash_alg == AES_GMAC && cipher_mode == GCM) ||
 136                 hash_alg == AES_CCM);
 137 #endif
 138 }
 139
 140 /** Set session authentication parameters */
 141 static int
 142 aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
 143                 struct aesni_mb_session *sess,
 144                 const struct rte_crypto_sym_xform *xform)
 145 {
 146         hash_one_block_t hash_oneblock_fn = NULL;
 147         unsigned int key_larger_block_size = 0;
 148         uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
 149         uint32_t auth_precompute = 1;
 150
 151         if (xform == NULL) {
 152                 sess->auth.algo = NULL_HASH;
 153                 return 0;
 154         }
 155
 156         if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
 157                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
 158                 return -1;
 159         }
 160
 161         /* Set IV parameters */
 162         sess->auth_iv.offset = xform->auth.iv.offset;
 163         sess->auth_iv.length = xform->auth.iv.length;
 164
 165         /* Set the request digest size */
 166         sess->auth.req_digest_len = xform->auth.digest_length;
 167
 168         /* Select auth generate/verify */
 169         sess->auth.operation = xform->auth.op;
 170
 171         /* Set Authentication Parameters */
 172         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
 173                 sess->auth.algo = AES_XCBC;
 174
 175                 uint16_t xcbc_mac_digest_len =
 176                         get_truncated_digest_byte_length(AES_XCBC);
 177                 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
 178                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 179                         return -EINVAL;
 180                 }
 181                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 182
 183                 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
 184                                 sess->auth.xcbc.k1_expanded,
 185                                 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
 186                 return 0;
 187         }
 188
 189         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
 190                 uint32_t dust[4*15];
 191
 192                 sess->auth.algo = AES_CMAC;
 193
 194                 uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
 195
 196                 if (sess->auth.req_digest_len > cmac_digest_len) {
 197                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 198                         return -EINVAL;
 199                 }
 200                 /*
 201                  * Multi-buffer lib supports digest sizes from 4 to 16 bytes
 202                  * in version 0.50 and sizes of 12 and 16 bytes,
 203                  * in version 0.49.
 204                  * If size requested is different, generate the full digest
 205                  * (16 bytes) in a temporary location and then memcpy
 206                  * the requested number of bytes.
 207                  */
 208                 if (sess->auth.req_digest_len < 4)
 209                         sess->auth.gen_digest_len = cmac_digest_len;
 210                 else
 211                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 212
 213                 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
 214                                 sess->auth.cmac.expkey, dust);
 215                 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
 216                                 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
 217                 return 0;
 218         }
 219
 220         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 221                 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 222                         sess->cipher.direction = ENCRYPT;
 223                         sess->chain_order = CIPHER_HASH;
 224                 } else
 225                         sess->cipher.direction = DECRYPT;
 226
 227                 sess->auth.algo = AES_GMAC;
 228                 if (sess->auth.req_digest_len > get_digest_byte_length(AES_GMAC)) {
 229                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 230                         return -EINVAL;
 231                 }
 232                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 233                 sess->iv.length = xform->auth.iv.length;
 234                 sess->iv.offset = xform->auth.iv.offset;
 235
 236                 switch (xform->auth.key.length) {
 237                 case AES_128_BYTES:
 238                         IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
 239                                 &sess->cipher.gcm_key);
 240                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 241                         break;
 242                 case AES_192_BYTES:
 243                         IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
 244                                 &sess->cipher.gcm_key);
 245                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 246                         break;
 247                 case AES_256_BYTES:
 248                         IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
 249                                 &sess->cipher.gcm_key);
 250                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 251                         break;
 252                 default:
 253                         RTE_LOG(ERR, PMD, "failed to parse test type\n");
 254                         return -EINVAL;
 255                 }
 256
 257                 return 0;
 258         }
 259
 260 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 261         if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 262                 sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;
 263                 uint16_t zuc_eia3_digest_len =
 264                         get_truncated_digest_byte_length(IMB_AUTH_ZUC_EIA3_BITLEN);
 265                 if (sess->auth.req_digest_len != zuc_eia3_digest_len) {
 266                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 267                         return -EINVAL;
 268                 }
 269                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 270
 271                 memcpy(sess->auth.zuc_auth_key, xform->auth.key.data, 16);
 272                 return 0;
 273         } else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 274                 sess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN;
 275                 uint16_t snow3g_uia2_digest_len =
 276                         get_truncated_digest_byte_length(IMB_AUTH_SNOW3G_UIA2_BITLEN);
 277                 if (sess->auth.req_digest_len != snow3g_uia2_digest_len) {
 278                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 279                         return -EINVAL;
 280                 }
 281                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 282
 283                 IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->auth.key.data,
 284                                         &sess->auth.pKeySched_snow3g_auth);
 285                 return 0;
 286         } else if (xform->auth.algo == RTE_CRYPTO_AUTH_KASUMI_F9) {
 287                 sess->auth.algo = IMB_AUTH_KASUMI_UIA1;
 288                 uint16_t kasumi_f9_digest_len =
 289                         get_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1);
 290                 if (sess->auth.req_digest_len != kasumi_f9_digest_len) {
 291                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 292                         return -EINVAL;
 293                 }
 294                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 295
 296                 IMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr, xform->auth.key.data,
 297                                         &sess->auth.pKeySched_kasumi_auth);
 298                 return 0;
 299         }
 300 #endif
 301
 302         switch (xform->auth.algo) {
 303         case RTE_CRYPTO_AUTH_MD5_HMAC:
 304                 sess->auth.algo = MD5;
 305                 hash_oneblock_fn = mb_mgr->md5_one_block;
 306                 break;
 307         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 308                 sess->auth.algo = SHA1;
 309                 hash_oneblock_fn = mb_mgr->sha1_one_block;
 310                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
 311                         IMB_SHA1(mb_mgr,
 312                                 xform->auth.key.data,
 313                                 xform->auth.key.length,
 314                                 hashed_key);
 315                         key_larger_block_size = 1;
 316                 }
 317                 break;
 318         case RTE_CRYPTO_AUTH_SHA1:
 319                 sess->auth.algo = PLAIN_SHA1;
 320                 auth_precompute = 0;
 321                 break;
 322         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 323                 sess->auth.algo = SHA_224;
 324                 hash_oneblock_fn = mb_mgr->sha224_one_block;
 325                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
 326                         IMB_SHA224(mb_mgr,
 327                                 xform->auth.key.data,
 328                                 xform->auth.key.length,
 329                                 hashed_key);
 330                         key_larger_block_size = 1;
 331                 }
 332                 break;
 333         case RTE_CRYPTO_AUTH_SHA224:
 334                 sess->auth.algo = PLAIN_SHA_224;
 335                 auth_precompute = 0;
 336                 break;
 337         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 338                 sess->auth.algo = SHA_256;
 339                 hash_oneblock_fn = mb_mgr->sha256_one_block;
 340                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
 341                         IMB_SHA256(mb_mgr,
 342                                 xform->auth.key.data,
 343                                 xform->auth.key.length,
 344                                 hashed_key);
 345                         key_larger_block_size = 1;
 346                 }
 347                 break;
 348         case RTE_CRYPTO_AUTH_SHA256:
 349                 sess->auth.algo = PLAIN_SHA_256;
 350                 auth_precompute = 0;
 351                 break;
 352         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 353                 sess->auth.algo = SHA_384;
 354                 hash_oneblock_fn = mb_mgr->sha384_one_block;
 355                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
 356                         IMB_SHA384(mb_mgr,
 357                                 xform->auth.key.data,
 358                                 xform->auth.key.length,
 359                                 hashed_key);
 360                         key_larger_block_size = 1;
 361                 }
 362                 break;
 363         case RTE_CRYPTO_AUTH_SHA384:
 364                 sess->auth.algo = PLAIN_SHA_384;
 365                 auth_precompute = 0;
 366                 break;
 367         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 368                 sess->auth.algo = SHA_512;
 369                 hash_oneblock_fn = mb_mgr->sha512_one_block;
 370                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
 371                         IMB_SHA512(mb_mgr,
 372                                 xform->auth.key.data,
 373                                 xform->auth.key.length,
 374                                 hashed_key);
 375                         key_larger_block_size = 1;
 376                 }
 377                 break;
 378         case RTE_CRYPTO_AUTH_SHA512:
 379                 sess->auth.algo = PLAIN_SHA_512;
 380                 auth_precompute = 0;
 381                 break;
 382         default:
 383                 AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
 384                 return -ENOTSUP;
 385         }
 386         uint16_t trunc_digest_size =
 387                         get_truncated_digest_byte_length(sess->auth.algo);
 388         uint16_t full_digest_size =
 389                         get_digest_byte_length(sess->auth.algo);
 390
 391         if (sess->auth.req_digest_len > full_digest_size ||
 392                         sess->auth.req_digest_len == 0) {
 393                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 394                 return -EINVAL;
 395         }
 396
 397         if (sess->auth.req_digest_len != trunc_digest_size &&
 398                         sess->auth.req_digest_len != full_digest_size)
 399                 sess->auth.gen_digest_len = full_digest_size;
 400         else
 401                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 402
 403         /* Plain SHA does not require precompute key */
 404         if (auth_precompute == 0)
 405                 return 0;
 406
 407         /* Calculate Authentication precomputes */
 408         if (key_larger_block_size) {
 409                 calculate_auth_precomputes(hash_oneblock_fn,
 410                         sess->auth.pads.inner, sess->auth.pads.outer,
 411                         hashed_key,
 412                         xform->auth.key.length,
 413                         get_auth_algo_blocksize(sess->auth.algo));
 414         } else {
 415                 calculate_auth_precomputes(hash_oneblock_fn,
 416                         sess->auth.pads.inner, sess->auth.pads.outer,
 417                         xform->auth.key.data,
 418                         xform->auth.key.length,
 419                         get_auth_algo_blocksize(sess->auth.algo));
 420         }
 421
 422         return 0;
 423 }
 424
 425 /** Set session cipher parameters */
 426 static int
 427 aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
 428                 struct aesni_mb_session *sess,
 429                 const struct rte_crypto_sym_xform *xform)
 430 {
 431         uint8_t is_aes = 0;
 432         uint8_t is_3DES = 0;
 433         uint8_t is_docsis = 0;
 434 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 435         uint8_t is_zuc = 0;
 436         uint8_t is_snow3g = 0;
 437         uint8_t is_kasumi = 0;
 438 #endif
 439
 440         if (xform == NULL) {
 441                 sess->cipher.mode = NULL_CIPHER;
 442                 return 0;
 443         }
 444
 445         if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
 446                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
 447                 return -EINVAL;
 448         }
 449
 450         /* Select cipher direction */
 451         switch (xform->cipher.op) {
 452         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
 453                 sess->cipher.direction = ENCRYPT;
 454                 break;
 455         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
 456                 sess->cipher.direction = DECRYPT;
 457                 break;
 458         default:
 459                 AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
 460                 return -EINVAL;
 461         }
 462
 463         /* Select cipher mode */
 464         switch (xform->cipher.algo) {
 465         case RTE_CRYPTO_CIPHER_AES_CBC:
 466                 sess->cipher.mode = CBC;
 467                 is_aes = 1;
 468                 break;
 469         case RTE_CRYPTO_CIPHER_AES_CTR:
 470                 sess->cipher.mode = CNTR;
 471                 is_aes = 1;
 472                 break;
 473         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 474                 sess->cipher.mode = DOCSIS_SEC_BPI;
 475                 is_docsis = 1;
 476                 break;
 477         case RTE_CRYPTO_CIPHER_DES_CBC:
 478                 sess->cipher.mode = DES;
 479                 break;
 480         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 481                 sess->cipher.mode = DOCSIS_DES;
 482                 break;
 483         case RTE_CRYPTO_CIPHER_3DES_CBC:
 484                 sess->cipher.mode = DES3;
 485                 is_3DES = 1;
 486                 break;
 487 #if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
 488         case RTE_CRYPTO_CIPHER_AES_ECB:
 489                 sess->cipher.mode = ECB;
 490                 is_aes = 1;
 491                 break;
 492 #endif
 493 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 494         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 495                 sess->cipher.mode = IMB_CIPHER_ZUC_EEA3;
 496                 is_zuc = 1;
 497                 break;
 498         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 499                 sess->cipher.mode = IMB_CIPHER_SNOW3G_UEA2_BITLEN;
 500                 is_snow3g = 1;
 501                 break;
 502         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 503                 sess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN;
 504                 is_kasumi = 1;
 505                 break;
 506 #endif
 507         default:
 508                 AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
 509                 return -ENOTSUP;
 510         }
 511
 512         /* Set IV parameters */
 513         sess->iv.offset = xform->cipher.iv.offset;
 514         sess->iv.length = xform->cipher.iv.length;
 515
 516         /* Check key length and choose key expansion function for AES */
 517         if (is_aes) {
 518                 switch (xform->cipher.key.length) {
 519                 case AES_128_BYTES:
 520                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 521                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 522                                         sess->cipher.expanded_aes_keys.encode,
 523                                         sess->cipher.expanded_aes_keys.decode);
 524                         break;
 525                 case AES_192_BYTES:
 526                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 527                         IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
 528                                         sess->cipher.expanded_aes_keys.encode,
 529                                         sess->cipher.expanded_aes_keys.decode);
 530                         break;
 531                 case AES_256_BYTES:
 532                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 533                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 534                                         sess->cipher.expanded_aes_keys.encode,
 535                                         sess->cipher.expanded_aes_keys.decode);
 536                         break;
 537                 default:
 538                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 539                         return -EINVAL;
 540                 }
 541         } else if (is_docsis) {
 542                 switch (xform->cipher.key.length) {
 543                 case AES_128_BYTES:
 544                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 545                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 546                                         sess->cipher.expanded_aes_keys.encode,
 547                                         sess->cipher.expanded_aes_keys.decode);
 548                         break;
 549 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 550                 case AES_256_BYTES:
 551                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 552                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 553                                         sess->cipher.expanded_aes_keys.encode,
 554                                         sess->cipher.expanded_aes_keys.decode);
 555                         break;
 556 #endif
 557                 default:
 558                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 559                         return -EINVAL;
 560                 }
 561         } else if (is_3DES) {
 562                 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
 563                                 sess->cipher.exp_3des_keys.key[1],
 564                                 sess->cipher.exp_3des_keys.key[2]};
 565
 566                 switch (xform->cipher.key.length) {
 567                 case  24:
 568                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 569                                         xform->cipher.key.data);
 570                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 571                                         xform->cipher.key.data + 8);
 572                         IMB_DES_KEYSCHED(mb_mgr, keys[2],
 573                                         xform->cipher.key.data + 16);
 574
 575                         /* Initialize keys - 24 bytes: [K1-K2-K3] */
 576                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 577                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 578                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
 579                         break;
 580                 case 16:
 581                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 582                                         xform->cipher.key.data);
 583                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 584                                         xform->cipher.key.data + 8);
 585                         /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
 586                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 587                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 588                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 589                         break;
 590                 case 8:
 591                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 592                                         xform->cipher.key.data);
 593
 594                         /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
 595                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 596                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
 597                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 598                         break;
 599                 default:
 600                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 601                         return -EINVAL;
 602                 }
 603
 604                 sess->cipher.key_length_in_bytes = 24;
 605 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 606         } else if (is_zuc) {
 607                 if (xform->cipher.key.length != 16) {
 608                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 609                         return -EINVAL;
 610                 }
 611                 sess->cipher.key_length_in_bytes = 16;
 612                 memcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data,
 613                         16);
 614         } else if (is_snow3g) {
 615                 if (xform->cipher.key.length != 16) {
 616                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 617                         return -EINVAL;
 618                 }
 619                 sess->cipher.key_length_in_bytes = 16;
 620                 IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->cipher.key.data,
 621                                         &sess->cipher.pKeySched_snow3g_cipher);
 622         } else if (is_kasumi) {
 623                 if (xform->cipher.key.length != 16) {
 624                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 625                         return -EINVAL;
 626                 }
 627                 sess->cipher.key_length_in_bytes = 16;
 628                 IMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data,
 629                                         &sess->cipher.pKeySched_kasumi_cipher);
 630 #endif
 631         } else {
 632                 if (xform->cipher.key.length != 8) {
 633                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 634                         return -EINVAL;
 635                 }
 636                 sess->cipher.key_length_in_bytes = 8;
 637
 638                 IMB_DES_KEYSCHED(mb_mgr,
 639                         (uint64_t *)sess->cipher.expanded_aes_keys.encode,
 640                                 xform->cipher.key.data);
 641                 IMB_DES_KEYSCHED(mb_mgr,
 642                         (uint64_t *)sess->cipher.expanded_aes_keys.decode,
 643                                 xform->cipher.key.data);
 644         }
 645
 646         return 0;
 647 }
 648
 649 static int
 650 aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
 651                 struct aesni_mb_session *sess,
 652                 const struct rte_crypto_sym_xform *xform)
 653 {
 654         switch (xform->aead.op) {
 655         case RTE_CRYPTO_AEAD_OP_ENCRYPT:
 656                 sess->cipher.direction = ENCRYPT;
 657                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 658                 break;
 659         case RTE_CRYPTO_AEAD_OP_DECRYPT:
 660                 sess->cipher.direction = DECRYPT;
 661                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 662                 break;
 663         default:
 664                 AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
 665                 return -EINVAL;
 666         }
 667
 668         /* Set IV parameters */
 669         sess->iv.offset = xform->aead.iv.offset;
 670         sess->iv.length = xform->aead.iv.length;
 671
 672         /* Set digest sizes */
 673         sess->auth.req_digest_len = xform->aead.digest_length;
 674         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 675
 676         switch (xform->aead.algo) {
 677         case RTE_CRYPTO_AEAD_AES_CCM:
 678                 sess->cipher.mode = CCM;
 679                 sess->auth.algo = AES_CCM;
 680
 681                 /* Check key length and choose key expansion function for AES */
 682                 switch (xform->aead.key.length) {
 683                 case AES_128_BYTES:
 684                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 685                         IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
 686                                         sess->cipher.expanded_aes_keys.encode,
 687                                         sess->cipher.expanded_aes_keys.decode);
 688                         break;
 689                 case AES_256_BYTES:
 690                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 691                         IMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data,
 692                                         sess->cipher.expanded_aes_keys.encode,
 693                                         sess->cipher.expanded_aes_keys.decode);
 694                         break;
 695                 default:
 696                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 697                         return -EINVAL;
 698                 }
 699
 700                 /* CCM digests must be between 4 and 16 and an even number */
 701                 if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
 702                                 sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
 703                                 (sess->auth.req_digest_len & 1) == 1) {
 704                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 705                         return -EINVAL;
 706                 }
 707                 break;
 708
 709         case RTE_CRYPTO_AEAD_AES_GCM:
 710                 sess->cipher.mode = GCM;
 711                 sess->auth.algo = AES_GMAC;
 712
 713                 switch (xform->aead.key.length) {
 714                 case AES_128_BYTES:
 715                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 716                         IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
 717                                 &sess->cipher.gcm_key);
 718                         break;
 719                 case AES_192_BYTES:
 720                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 721                         IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
 722                                 &sess->cipher.gcm_key);
 723                         break;
 724                 case AES_256_BYTES:
 725                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 726                         IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
 727                                 &sess->cipher.gcm_key);
 728                         break;
 729                 default:
 730                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 731                         return -EINVAL;
 732                 }
 733
 734                 /* GCM digest size must be between 1 and 16 */
 735                 if (sess->auth.req_digest_len == 0 ||
 736                                 sess->auth.req_digest_len > 16) {
 737                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 738                         return -EINVAL;
 739                 }
 740                 break;
 741
 742 #if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
 743         case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
 744                 sess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305;
 745                 sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;
 746
 747                 if (xform->aead.key.length != 32) {
 748                         AESNI_MB_LOG(ERR, "Invalid key length");
 749                         return -EINVAL;
 750                 }
 751                 sess->cipher.key_length_in_bytes = 32;
 752                 memcpy(sess->cipher.expanded_aes_keys.encode,
 753                         xform->aead.key.data, 32);
 754                 if (sess->auth.req_digest_len != 16) {
 755                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 756                         return -EINVAL;
 757                 }
 758                 break;
 759 #endif
 760         default:
 761                 AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
 762                 return -ENOTSUP;
 763         }
 764
 765         return 0;
 766 }
 767
 768 /** Parse crypto xform chain and set private session parameters */
 769 int
 770 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 771                 struct aesni_mb_session *sess,
 772                 const struct rte_crypto_sym_xform *xform)
 773 {
 774         const struct rte_crypto_sym_xform *auth_xform = NULL;
 775         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 776         const struct rte_crypto_sym_xform *aead_xform = NULL;
 777         int ret;
 778
 779         /* Select Crypto operation - hash then cipher / cipher then hash */
 780         switch (aesni_mb_get_chain_order(xform)) {
 781         case AESNI_MB_OP_HASH_CIPHER:
 782                 sess->chain_order = HASH_CIPHER;
 783                 auth_xform = xform;
 784                 cipher_xform = xform->next;
 785                 break;
 786         case AESNI_MB_OP_CIPHER_HASH:
 787                 sess->chain_order = CIPHER_HASH;
 788                 auth_xform = xform->next;
 789                 cipher_xform = xform;
 790                 break;
 791         case AESNI_MB_OP_HASH_ONLY:
 792                 sess->chain_order = HASH_CIPHER;
 793                 auth_xform = xform;
 794                 cipher_xform = NULL;
 795                 break;
 796         case AESNI_MB_OP_CIPHER_ONLY:
 797                 /*
 798                  * Multi buffer library operates only at two modes,
 799                  * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
 800                  * chain order depends on cipher operation: encryption is always
 801                  * the first operation and decryption the last one.
 802                  */
 803                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 804                         sess->chain_order = CIPHER_HASH;
 805                 else
 806                         sess->chain_order = HASH_CIPHER;
 807                 auth_xform = NULL;
 808                 cipher_xform = xform;
 809                 break;
 810         case AESNI_MB_OP_AEAD_CIPHER_HASH:
 811                 sess->chain_order = CIPHER_HASH;
 812                 sess->aead.aad_len = xform->aead.aad_length;
 813                 aead_xform = xform;
 814                 break;
 815         case AESNI_MB_OP_AEAD_HASH_CIPHER:
 816                 sess->chain_order = HASH_CIPHER;
 817                 sess->aead.aad_len = xform->aead.aad_length;
 818                 aead_xform = xform;
 819                 break;
 820         case AESNI_MB_OP_NOT_SUPPORTED:
 821         default:
 822                 AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
 823                 return -ENOTSUP;
 824         }
 825
 826         /* Default IV length = 0 */
 827         sess->iv.length = 0;
 828         sess->auth_iv.length = 0;
 829
 830         ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
 831         if (ret != 0) {
 832                 AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
 833                 return ret;
 834         }
 835
 836         ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
 837                         cipher_xform);
 838         if (ret != 0) {
 839                 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 840                 return ret;
 841         }
 842
 843         if (aead_xform) {
 844                 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
 845                                 aead_xform);
 846                 if (ret != 0) {
 847                         AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
 848                         return ret;
 849                 }
 850         }
 851
 852         return 0;
 853 }
 854
 855 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 856 /** Check DOCSIS security session configuration is valid */
 857 static int
 858 check_docsis_sec_session(struct rte_security_session_conf *conf)
 859 {
 860         struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;
 861         struct rte_security_docsis_xform *docsis = &conf->docsis;
 862
 863         /* Downlink: CRC generate -> Cipher encrypt */
 864         if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
 865
 866                 if (crypto_sym != NULL &&
 867                     crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 868                     crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
 869                     crypto_sym->cipher.algo ==
 870                                         RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
 871                     (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||
 872                      crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&
 873                     crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
 874                     crypto_sym->next == NULL) {
 875                         return 0;
 876                 }
 877         /* Uplink: Cipher decrypt -> CRC verify */
 878         } else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {
 879
 880                 if (crypto_sym != NULL &&
 881                     crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 882                     crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
 883                     crypto_sym->cipher.algo ==
 884                                         RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
 885                     (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||
 886                      crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&
 887                     crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
 888                     crypto_sym->next == NULL) {
 889                         return 0;
 890                 }
 891         }
 892
 893         return -EINVAL;
 894 }
 895
 896 /** Set DOCSIS security session auth (CRC) parameters */
 897 static int
 898 aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
 899                 struct rte_security_docsis_xform *xform)
 900 {
 901         if (xform == NULL) {
 902                 AESNI_MB_LOG(ERR, "Invalid DOCSIS xform");
 903                 return -EINVAL;
 904         }
 905
 906         /* Select CRC generate/verify */
 907         if (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) {
 908                 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
 909                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 910         } else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
 911                 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
 912                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 913         } else {
 914                 AESNI_MB_LOG(ERR, "Unsupported DOCSIS direction");
 915                 return -ENOTSUP;
 916         }
 917
 918         sess->auth.req_digest_len = RTE_ETHER_CRC_LEN;
 919         sess->auth.gen_digest_len = RTE_ETHER_CRC_LEN;
 920
 921         return 0;
 922 }
 923
 924 /**
 925  * Parse DOCSIS security session configuration and set private session
 926  * parameters
 927  */
 928 int
 929 aesni_mb_set_docsis_sec_session_parameters(
 930                 __rte_unused struct rte_cryptodev *dev,
 931                 struct rte_security_session_conf *conf,
 932                 void *sess)
 933 {
 934         struct rte_security_docsis_xform *docsis_xform;
 935         struct rte_crypto_sym_xform *cipher_xform;
 936         struct aesni_mb_session *aesni_sess = sess;
 937         struct aesni_mb_private *internals = dev->data->dev_private;
 938         int ret;
 939
 940         ret = check_docsis_sec_session(conf);
 941         if (ret) {
 942                 AESNI_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
 943                 return ret;
 944         }
 945
 946         switch (conf->docsis.direction) {
 947         case RTE_SECURITY_DOCSIS_UPLINK:
 948                 aesni_sess->chain_order = IMB_ORDER_CIPHER_HASH;
 949                 docsis_xform = &conf->docsis;
 950                 cipher_xform = conf->crypto_xform;
 951                 break;
 952         case RTE_SECURITY_DOCSIS_DOWNLINK:
 953                 aesni_sess->chain_order = IMB_ORDER_HASH_CIPHER;
 954                 cipher_xform = conf->crypto_xform;
 955                 docsis_xform = &conf->docsis;
 956                 break;
 957         default:
 958                 return -EINVAL;
 959         }
 960
 961         /* Default IV length = 0 */
 962         aesni_sess->iv.length = 0;
 963
 964         ret = aesni_mb_set_docsis_sec_session_auth_parameters(aesni_sess,
 965                         docsis_xform);
 966         if (ret != 0) {
 967                 AESNI_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
 968                 return -EINVAL;
 969         }
 970
 971         ret = aesni_mb_set_session_cipher_parameters(internals->mb_mgr,
 972                         aesni_sess, cipher_xform);
 973
 974         if (ret != 0) {
 975                 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 976                 return -EINVAL;
 977         }
 978
 979         return 0;
 980 }
 981 #endif
 982
 983 /**
 984  * burst enqueue, place crypto operations on ingress queue for processing.
 985  *
 986  * @param __qp         Queue Pair to process
 987  * @param ops          Crypto operations for processing
 988  * @param nb_ops       Number of crypto operations for processing
 989  *
 990  * @return
 991  * - Number of crypto operations enqueued
 992  */
 993 static uint16_t
 994 aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
 995                 uint16_t nb_ops)
 996 {
 997         struct aesni_mb_qp *qp = __qp;
 998
 999         unsigned int nb_enqueued;
1000
1001         nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
1002                         (void **)ops, nb_ops, NULL);
1003
1004         qp->stats.enqueued_count += nb_enqueued;
1005
1006         return nb_enqueued;
1007 }
1008
1009 /** Get multi buffer session */
1010 static inline struct aesni_mb_session *
1011 get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
1012 {
1013         struct aesni_mb_session *sess = NULL;
1014
1015         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
1016                 if (likely(op->sym->session != NULL))
1017                         sess = (struct aesni_mb_session *)
1018                                         get_sym_session_private_data(
1019                                         op->sym->session,
1020                                         cryptodev_driver_id);
1021 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1022         } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1023                 if (likely(op->sym->sec_session != NULL))
1024                         sess = (struct aesni_mb_session *)
1025                                         get_sec_session_private_data(
1026                                                 op->sym->sec_session);
1027 #endif
1028         } else {
1029                 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
1030                 void *_sess_private_data = NULL;
1031
1032                 if (_sess == NULL)
1033                         return NULL;
1034
1035                 if (rte_mempool_get(qp->sess_mp_priv,
1036                                 (void **)&_sess_private_data))
1037                         return NULL;
1038
1039                 sess = (struct aesni_mb_session *)_sess_private_data;
1040
1041                 if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
1042                                 sess, op->sym->xform) != 0)) {
1043                         rte_mempool_put(qp->sess_mp, _sess);
1044                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
1045                         sess = NULL;
1046                 }
1047                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
1048                 set_sym_session_private_data(op->sym->session,
1049                                 cryptodev_driver_id, _sess_private_data);
1050         }
1051
1052         if (unlikely(sess == NULL))
1053                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1054
1055         return sess;
1056 }
1057
1058 static inline uint64_t
1059 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
1060                 uint32_t oop)
1061 {
1062         struct rte_mbuf *m_src, *m_dst;
1063         uint8_t *p_src, *p_dst;
1064         uintptr_t u_src, u_dst;
1065         uint32_t cipher_end, auth_end;
1066
1067         /* Only cipher then hash needs special calculation. */
1068         if (!oop || session->chain_order != CIPHER_HASH)
1069                 return op->sym->auth.data.offset;
1070
1071         m_src = op->sym->m_src;
1072         m_dst = op->sym->m_dst;
1073
1074         p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
1075         p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
1076         u_src = (uintptr_t)p_src;
1077         u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
1078
1079         /**
1080          * Copy the content between cipher offset and auth offset for generating
1081          * correct digest.
1082          */
1083         if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
1084                 memcpy(p_dst + op->sym->auth.data.offset,
1085                                 p_src + op->sym->auth.data.offset,
1086                                 op->sym->cipher.data.offset -
1087                                 op->sym->auth.data.offset);
1088
1089         /**
1090          * Copy the content between (cipher offset + length) and (auth offset +
1091          * length) for generating correct digest
1092          */
1093         cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
1094         auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
1095         if (cipher_end < auth_end)
1096                 memcpy(p_dst + cipher_end, p_src + cipher_end,
1097                                 auth_end - cipher_end);
1098
1099         /**
1100          * Since intel-ipsec-mb only supports positive values,
1101          * we need to deduct the correct offset between src and dst.
1102          */
1103
1104         return u_src < u_dst ? (u_dst - u_src) :
1105                         (UINT64_MAX - u_src + u_dst + 1);
1106 }
1107
1108 static inline void
1109 set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
1110                 union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
1111                 void *iv, void *aad, void *digest, void *udata)
1112 {
1113         /* Set crypto operation */
1114         job->chain_order = session->chain_order;
1115
1116         /* Set cipher parameters */
1117         job->cipher_direction = session->cipher.direction;
1118         job->cipher_mode = session->cipher.mode;
1119
1120         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
1121
1122         /* Set authentication parameters */
1123         job->hash_alg = session->auth.algo;
1124         job->iv = iv;
1125
1126         switch (job->hash_alg) {
1127         case AES_XCBC:
1128                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
1129                 job->u.XCBC._k2 = session->auth.xcbc.k2;
1130                 job->u.XCBC._k3 = session->auth.xcbc.k3;
1131
1132                 job->aes_enc_key_expanded =
1133                                 session->cipher.expanded_aes_keys.encode;
1134                 job->aes_dec_key_expanded =
1135                                 session->cipher.expanded_aes_keys.decode;
1136                 break;
1137
1138         case AES_CCM:
1139                 job->u.CCM.aad = (uint8_t *)aad + 18;
1140                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1141                 job->aes_enc_key_expanded =
1142                                 session->cipher.expanded_aes_keys.encode;
1143                 job->aes_dec_key_expanded =
1144                                 session->cipher.expanded_aes_keys.decode;
1145                 job->iv++;
1146                 break;
1147
1148         case AES_CMAC:
1149                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1150                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1151                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1152                 job->aes_enc_key_expanded =
1153                                 session->cipher.expanded_aes_keys.encode;
1154                 job->aes_dec_key_expanded =
1155                                 session->cipher.expanded_aes_keys.decode;
1156                 break;
1157
1158         case AES_GMAC:
1159                 if (session->cipher.mode == GCM) {
1160                         job->u.GCM.aad = aad;
1161                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1162                 } else {
1163                         /* For GMAC */
1164                         job->u.GCM.aad = buf;
1165                         job->u.GCM.aad_len_in_bytes = len;
1166                         job->cipher_mode = GCM;
1167                 }
1168                 job->aes_enc_key_expanded = &session->cipher.gcm_key;
1169                 job->aes_dec_key_expanded = &session->cipher.gcm_key;
1170                 break;
1171
1172         default:
1173                 job->u.HMAC._hashed_auth_key_xor_ipad =
1174                                 session->auth.pads.inner;
1175                 job->u.HMAC._hashed_auth_key_xor_opad =
1176                                 session->auth.pads.outer;
1177
1178                 if (job->cipher_mode == DES3) {
1179                         job->aes_enc_key_expanded =
1180                                 session->cipher.exp_3des_keys.ks_ptr;
1181                         job->aes_dec_key_expanded =
1182                                 session->cipher.exp_3des_keys.ks_ptr;
1183                 } else {
1184                         job->aes_enc_key_expanded =
1185                                 session->cipher.expanded_aes_keys.encode;
1186                         job->aes_dec_key_expanded =
1187                                 session->cipher.expanded_aes_keys.decode;
1188                 }
1189         }
1190
1191         /*
1192          * Multi-buffer library current only support returning a truncated
1193          * digest length as specified in the relevant IPsec RFCs
1194          */
1195
1196         /* Set digest location and length */
1197         job->auth_tag_output = digest;
1198         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1199
1200         /* Set IV parameters */
1201         job->iv_len_in_bytes = session->iv.length;
1202
1203         /* Data Parameters */
1204         job->src = buf;
1205         job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;
1206         job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;
1207         job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;
1208         if (job->hash_alg == AES_GMAC && session->cipher.mode != GCM) {
1209                 job->msg_len_to_hash_in_bytes = 0;
1210                 job->msg_len_to_cipher_in_bytes = 0;
1211         } else {
1212                 job->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -
1213                         sofs.ofs.auth.tail;
1214                 job->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -
1215                         sofs.ofs.cipher.tail;
1216         }
1217
1218         job->user_data = udata;
1219 }
1220
1221 /**
1222  * Process a crypto operation and complete a JOB_AES_HMAC job structure for
1223  * submission to the multi buffer library for processing.
1224  *
1225  * @param       qp      queue pair
1226  * @param       job     JOB_AES_HMAC structure to fill
1227  * @param       m       mbuf to process
1228  *
1229  * @return
1230  * - Completed JOB_AES_HMAC structure pointer on success
1231  * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
1232  */
1233 static inline int
1234 set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
1235                 struct rte_crypto_op *op, uint8_t *digest_idx)
1236 {
1237         struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
1238         struct aesni_mb_session *session;
1239         uint32_t m_offset, oop;
1240
1241         session = get_session(qp, op);
1242         if (session == NULL) {
1243                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1244                 return -1;
1245         }
1246
1247         /* Set crypto operation */
1248         job->chain_order = session->chain_order;
1249
1250         /* Set cipher parameters */
1251         job->cipher_direction = session->cipher.direction;
1252         job->cipher_mode = session->cipher.mode;
1253
1254         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
1255
1256         /* Set authentication parameters */
1257         job->hash_alg = session->auth.algo;
1258
1259         const int aead = is_aead_algo(job->hash_alg, job->cipher_mode);
1260
1261         switch (job->hash_alg) {
1262         case AES_XCBC:
1263                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
1264                 job->u.XCBC._k2 = session->auth.xcbc.k2;
1265                 job->u.XCBC._k3 = session->auth.xcbc.k3;
1266
1267                 job->aes_enc_key_expanded =
1268                                 session->cipher.expanded_aes_keys.encode;
1269                 job->aes_dec_key_expanded =
1270                                 session->cipher.expanded_aes_keys.decode;
1271                 break;
1272
1273         case AES_CCM:
1274                 job->u.CCM.aad = op->sym->aead.aad.data + 18;
1275                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1276                 job->aes_enc_key_expanded =
1277                                 session->cipher.expanded_aes_keys.encode;
1278                 job->aes_dec_key_expanded =
1279                                 session->cipher.expanded_aes_keys.decode;
1280                 break;
1281
1282         case AES_CMAC:
1283                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1284                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1285                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1286                 job->aes_enc_key_expanded =
1287                                 session->cipher.expanded_aes_keys.encode;
1288                 job->aes_dec_key_expanded =
1289                                 session->cipher.expanded_aes_keys.decode;
1290                 break;
1291
1292         case AES_GMAC:
1293                 if (session->cipher.mode == GCM) {
1294                         job->u.GCM.aad = op->sym->aead.aad.data;
1295                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1296                 } else {
1297                         /* For GMAC */
1298                         job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
1299                                         uint8_t *, op->sym->auth.data.offset);
1300                         job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
1301                         job->cipher_mode = GCM;
1302                 }
1303                 job->aes_enc_key_expanded = &session->cipher.gcm_key;
1304                 job->aes_dec_key_expanded = &session->cipher.gcm_key;
1305                 break;
1306 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
1307         case IMB_AUTH_ZUC_EIA3_BITLEN:
1308                 job->u.ZUC_EIA3._key = session->auth.zuc_auth_key;
1309                 job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1310                                                 session->auth_iv.offset);
1311                 break;
1312         case IMB_AUTH_SNOW3G_UIA2_BITLEN:
1313                 job->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth;
1314                 job->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1315                                                 session->auth_iv.offset);
1316                 break;
1317         case IMB_AUTH_KASUMI_UIA1:
1318                 job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth;
1319                 break;
1320 #endif
1321 #if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
1322         case IMB_AUTH_CHACHA20_POLY1305:
1323                 job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
1324                 job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
1325                 job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
1326                 job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
1327                 break;
1328 #endif
1329         default:
1330                 job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
1331                 job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
1332
1333                 if (job->cipher_mode == DES3) {
1334                         job->aes_enc_key_expanded =
1335                                 session->cipher.exp_3des_keys.ks_ptr;
1336                         job->aes_dec_key_expanded =
1337                                 session->cipher.exp_3des_keys.ks_ptr;
1338                 } else {
1339                         job->aes_enc_key_expanded =
1340                                 session->cipher.expanded_aes_keys.encode;
1341                         job->aes_dec_key_expanded =
1342                                 session->cipher.expanded_aes_keys.decode;
1343                 }
1344         }
1345
1346         if (aead)
1347                 m_offset = op->sym->aead.data.offset;
1348         else
1349                 m_offset = op->sym->cipher.data.offset;
1350
1351 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
1352         if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
1353                 job->aes_enc_key_expanded = session->cipher.zuc_cipher_key;
1354                 job->aes_dec_key_expanded = session->cipher.zuc_cipher_key;
1355         } else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
1356                 job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
1357                 m_offset = 0;
1358         } else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) {
1359                 job->enc_keys = &session->cipher.pKeySched_kasumi_cipher;
1360                 m_offset = 0;
1361         }
1362 #endif
1363
1364         if (!op->sym->m_dst) {
1365                 /* in-place operation */
1366                 m_dst = m_src;
1367                 oop = 0;
1368         } else if (op->sym->m_dst == op->sym->m_src) {
1369                 /* in-place operation */
1370                 m_dst = m_src;
1371                 oop = 0;
1372         } else {
1373                 /* out-of-place operation */
1374                 m_dst = op->sym->m_dst;
1375                 oop = 1;
1376         }
1377
1378         /* Set digest output location */
1379         if (job->hash_alg != NULL_HASH &&
1380                         session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1381                 job->auth_tag_output = qp->temp_digests[*digest_idx];
1382                 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1383         } else {
1384                 if (aead)
1385                         job->auth_tag_output = op->sym->aead.digest.data;
1386                 else
1387                         job->auth_tag_output = op->sym->auth.digest.data;
1388
1389                 if (session->auth.req_digest_len != session->auth.gen_digest_len) {
1390                         job->auth_tag_output = qp->temp_digests[*digest_idx];
1391                         *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1392                 }
1393         }
1394         /*
1395          * Multi-buffer library current only support returning a truncated
1396          * digest length as specified in the relevant IPsec RFCs
1397          */
1398
1399         /* Set digest length */
1400         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1401
1402         /* Set IV parameters */
1403         job->iv_len_in_bytes = session->iv.length;
1404
1405         /* Data Parameters */
1406         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1407         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
1408
1409         switch (job->hash_alg) {
1410         case AES_CCM:
1411                 job->cipher_start_src_offset_in_bytes =
1412                                 op->sym->aead.data.offset;
1413                 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
1414                 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
1415                 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
1416
1417                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1418                         session->iv.offset + 1);
1419                 break;
1420
1421         case AES_GMAC:
1422                 if (session->cipher.mode == GCM) {
1423                         job->cipher_start_src_offset_in_bytes =
1424                                         op->sym->aead.data.offset;
1425                         job->hash_start_src_offset_in_bytes =
1426                                         op->sym->aead.data.offset;
1427                         job->msg_len_to_cipher_in_bytes =
1428                                         op->sym->aead.data.length;
1429                         job->msg_len_to_hash_in_bytes =
1430                                         op->sym->aead.data.length;
1431                 } else {
1432                         job->cipher_start_src_offset_in_bytes =
1433                                         op->sym->auth.data.offset;
1434                         job->hash_start_src_offset_in_bytes =
1435                                         op->sym->auth.data.offset;
1436                         job->msg_len_to_cipher_in_bytes = 0;
1437                         job->msg_len_to_hash_in_bytes = 0;
1438                 }
1439
1440                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1441                                 session->iv.offset);
1442                 break;
1443
1444 #if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
1445         case IMB_AUTH_CHACHA20_POLY1305:
1446                 job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset;
1447                 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
1448                 job->msg_len_to_cipher_in_bytes =
1449                                 op->sym->aead.data.length;
1450                 job->msg_len_to_hash_in_bytes =
1451                                         op->sym->aead.data.length;
1452
1453                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1454                                 session->iv.offset);
1455                 break;
1456 #endif
1457         default:
1458                 /* For SNOW3G, length and offsets are already in bits */
1459                 job->cipher_start_src_offset_in_bytes =
1460                                 op->sym->cipher.data.offset;
1461                 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
1462
1463                 job->hash_start_src_offset_in_bytes = auth_start_offset(op,
1464                                 session, oop);
1465                 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
1466
1467                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1468                         session->iv.offset);
1469         }
1470
1471 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
1472         if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
1473                 job->msg_len_to_cipher_in_bytes >>= 3;
1474         else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)
1475                 job->msg_len_to_hash_in_bytes >>= 3;
1476 #endif
1477
1478         /* Set user data to be crypto operation data struct */
1479         job->user_data = op;
1480
1481         return 0;
1482 }
1483
1484 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1485 /**
1486  * Process a crypto operation containing a security op and complete a
1487  * JOB_AES_HMAC job structure for submission to the multi buffer library for
1488  * processing.
1489  */
1490 static inline int
1491 set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
1492                 struct rte_crypto_op *op, uint8_t *digest_idx)
1493 {
1494         struct rte_mbuf *m_src, *m_dst;
1495         struct rte_crypto_sym_op *sym;
1496         struct aesni_mb_session *session;
1497
1498         session = get_session(qp, op);
1499         if (unlikely(session == NULL)) {
1500                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1501                 return -1;
1502         }
1503
1504         /* Only DOCSIS protocol operations supported now */
1505         if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||
1506                         session->auth.algo != IMB_AUTH_DOCSIS_CRC32) {
1507                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1508                 return -1;
1509         }
1510
1511         sym = op->sym;
1512         m_src = sym->m_src;
1513
1514         if (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {
1515                 /* in-place operation */
1516                 m_dst = m_src;
1517         } else {
1518                 /* out-of-place operation not supported */
1519                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1520                 return -ENOTSUP;
1521         }
1522
1523         /* Set crypto operation */
1524         job->chain_order = session->chain_order;
1525
1526         /* Set cipher parameters */
1527         job->cipher_direction = session->cipher.direction;
1528         job->cipher_mode = session->cipher.mode;
1529
1530         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
1531         job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
1532         job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
1533
1534         /* Set IV parameters */
1535         job->iv_len_in_bytes = session->iv.length;
1536         job->iv = (uint8_t *)op + session->iv.offset;
1537
1538         /* Set authentication parameters */
1539         job->hash_alg = session->auth.algo;
1540
1541         /* Set digest output location */
1542         job->auth_tag_output = qp->temp_digests[*digest_idx];
1543         *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1544
1545         /* Set digest length */
1546         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1547
1548         /* Set data parameters */
1549         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1550         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,
1551                                                 sym->cipher.data.offset);
1552
1553         job->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;
1554         job->msg_len_to_cipher_in_bytes = sym->cipher.data.length;
1555
1556         job->hash_start_src_offset_in_bytes = sym->auth.data.offset;
1557         job->msg_len_to_hash_in_bytes = sym->auth.data.length;
1558
1559         job->user_data = op;
1560
1561         return 0;
1562 }
1563
1564 static inline void
1565 verify_docsis_sec_crc(JOB_AES_HMAC *job, uint8_t *status)
1566 {
1567         uint16_t crc_offset;
1568         uint8_t *crc;
1569
1570         if (!job->msg_len_to_hash_in_bytes)
1571                 return;
1572
1573         crc_offset = job->hash_start_src_offset_in_bytes +
1574                         job->msg_len_to_hash_in_bytes -
1575                         job->cipher_start_src_offset_in_bytes;
1576         crc = job->dst + crc_offset;
1577
1578         /* Verify CRC (at the end of the message) */
1579         if (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0)
1580                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1581 }
1582 #endif
1583
1584 static inline void
1585 verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
1586 {
1587         /* Verify digest if required */
1588         if (memcmp(job->auth_tag_output, digest, len) != 0)
1589                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1590 }
1591
1592 static inline void
1593 generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
1594                 struct aesni_mb_session *sess)
1595 {
1596         /* No extra copy needed */
1597         if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
1598                 return;
1599
1600         /*
1601          * This can only happen for HMAC, so only digest
1602          * for authentication algos is required
1603          */
1604         memcpy(op->sym->auth.digest.data, job->auth_tag_output,
1605                         sess->auth.req_digest_len);
1606 }
1607
1608 /**
1609  * Process a completed job and return rte_mbuf which job processed
1610  *
1611  * @param qp            Queue Pair to process
1612  * @param job   JOB_AES_HMAC job to process
1613  *
1614  * @return
1615  * - Returns processed crypto operation.
1616  * - Returns NULL on invalid job
1617  */
1618 static inline struct rte_crypto_op *
1619 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
1620 {
1621         struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
1622         struct aesni_mb_session *sess = NULL;
1623
1624 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1625         uint8_t is_docsis_sec = 0;
1626
1627         if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1628                 /*
1629                  * Assuming at this point that if it's a security type op, that
1630                  * this is for DOCSIS
1631                  */
1632                 is_docsis_sec = 1;
1633                 sess = get_sec_session_private_data(op->sym->sec_session);
1634         } else
1635 #endif
1636         {
1637                 sess = get_sym_session_private_data(op->sym->session,
1638                                                 cryptodev_driver_id);
1639         }
1640
1641         if (unlikely(sess == NULL)) {
1642                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1643                 return op;
1644         }
1645
1646         if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
1647                 switch (job->status) {
1648                 case STS_COMPLETED:
1649                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1650
1651                         if (job->hash_alg == NULL_HASH)
1652                                 break;
1653
1654                         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1655                                 if (is_aead_algo(job->hash_alg, sess->cipher.mode))
1656                                         verify_digest(job,
1657                                                 op->sym->aead.digest.data,
1658                                                 sess->auth.req_digest_len,
1659                                                 &op->status);
1660 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1661                                 else if (is_docsis_sec)
1662                                         verify_docsis_sec_crc(job,
1663                                                 &op->status);
1664 #endif
1665                                 else
1666                                         verify_digest(job,
1667                                                 op->sym->auth.digest.data,
1668                                                 sess->auth.req_digest_len,
1669                                                 &op->status);
1670                         } else
1671                                 generate_digest(job, op, sess);
1672                         break;
1673                 default:
1674                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1675                 }
1676         }
1677
1678         /* Free session if a session-less crypto op */
1679         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1680                 memset(sess, 0, sizeof(struct aesni_mb_session));
1681                 memset(op->sym->session, 0,
1682                         rte_cryptodev_sym_get_existing_header_session_size(
1683                                 op->sym->session));
1684                 rte_mempool_put(qp->sess_mp_priv, sess);
1685                 rte_mempool_put(qp->sess_mp, op->sym->session);
1686                 op->sym->session = NULL;
1687         }
1688
1689         return op;
1690 }
1691
1692 static inline void
1693 post_process_mb_sync_job(JOB_AES_HMAC *job)
1694 {
1695         uint32_t *st;
1696
1697         st = job->user_data;
1698         st[0] = (job->status == STS_COMPLETED) ? 0 : EBADMSG;
1699 }
1700
1701 /**
1702  * Process a completed JOB_AES_HMAC job and keep processing jobs until
1703  * get_completed_job return NULL
1704  *
1705  * @param qp            Queue Pair to process
1706  * @param job           JOB_AES_HMAC job
1707  *
1708  * @return
1709  * - Number of processed jobs
1710  */
1711 static unsigned
1712 handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
1713                 struct rte_crypto_op **ops, uint16_t nb_ops)
1714 {
1715         struct rte_crypto_op *op = NULL;
1716         unsigned processed_jobs = 0;
1717
1718         while (job != NULL) {
1719                 op = post_process_mb_job(qp, job);
1720
1721                 if (op) {
1722                         ops[processed_jobs++] = op;
1723                         qp->stats.dequeued_count++;
1724                 } else {
1725                         qp->stats.dequeue_err_count++;
1726                         break;
1727                 }
1728                 if (processed_jobs == nb_ops)
1729                         break;
1730
1731                 job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
1732         }
1733
1734         return processed_jobs;
1735 }
1736
1737 static inline uint32_t
1738 handle_completed_sync_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr)
1739 {
1740         uint32_t i;
1741
1742         for (i = 0; job != NULL; i++, job = IMB_GET_COMPLETED_JOB(mb_mgr))
1743                 post_process_mb_sync_job(job);
1744
1745         return i;
1746 }
1747
1748 static inline uint32_t
1749 flush_mb_sync_mgr(MB_MGR *mb_mgr)
1750 {
1751         JOB_AES_HMAC *job;
1752
1753         job = IMB_FLUSH_JOB(mb_mgr);
1754         return handle_completed_sync_jobs(job, mb_mgr);
1755 }
1756
1757 static inline uint16_t
1758 flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
1759                 uint16_t nb_ops)
1760 {
1761         int processed_ops = 0;
1762
1763         /* Flush the remaining jobs */
1764         JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
1765
1766         if (job)
1767                 processed_ops += handle_completed_jobs(qp, job,
1768                                 &ops[processed_ops], nb_ops - processed_ops);
1769
1770         return processed_ops;
1771 }
1772
1773 static inline JOB_AES_HMAC *
1774 set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
1775 {
1776         job->chain_order = HASH_CIPHER;
1777         job->cipher_mode = NULL_CIPHER;
1778         job->hash_alg = NULL_HASH;
1779         job->cipher_direction = DECRYPT;
1780
1781         /* Set user data to be crypto operation data struct */
1782         job->user_data = op;
1783
1784         return job;
1785 }
1786
1787 static uint16_t
1788 aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1789                 uint16_t nb_ops)
1790 {
1791         struct aesni_mb_qp *qp = queue_pair;
1792
1793         struct rte_crypto_op *op;
1794         JOB_AES_HMAC *job;
1795
1796         int retval, processed_jobs = 0;
1797
1798         if (unlikely(nb_ops == 0))
1799                 return 0;
1800
1801         uint8_t digest_idx = qp->digest_idx;
1802         do {
1803                 /* Get next free mb job struct from mb manager */
1804                 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1805                 if (unlikely(job == NULL)) {
1806                         /* if no free mb job structs we need to flush mb_mgr */
1807                         processed_jobs += flush_mb_mgr(qp,
1808                                         &ops[processed_jobs],
1809                                         nb_ops - processed_jobs);
1810
1811                         if (nb_ops == processed_jobs)
1812                                 break;
1813
1814                         job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1815                 }
1816
1817                 /*
1818                  * Get next operation to process from ingress queue.
1819                  * There is no need to return the job to the MB_MGR
1820                  * if there are no more operations to process, since the MB_MGR
1821                  * can use that pointer again in next get_next calls.
1822                  */
1823                 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1824                 if (retval < 0)
1825                         break;
1826
1827 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1828                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1829                         retval = set_sec_mb_job_params(job, qp, op,
1830                                                 &digest_idx);
1831                 else
1832 #endif
1833                         retval = set_mb_job_params(job, qp, op, &digest_idx);
1834
1835                 if (unlikely(retval != 0)) {
1836                         qp->stats.dequeue_err_count++;
1837                         set_job_null_op(job, op);
1838                 }
1839
1840                 /* Submit job to multi-buffer for processing */
1841 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1842                 job = IMB_SUBMIT_JOB(qp->mb_mgr);
1843 #else
1844                 job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
1845 #endif
1846                 /*
1847                  * If submit returns a processed job then handle it,
1848                  * before submitting subsequent jobs
1849                  */
1850                 if (job)
1851                         processed_jobs += handle_completed_jobs(qp, job,
1852                                         &ops[processed_jobs],
1853                                         nb_ops - processed_jobs);
1854
1855         } while (processed_jobs < nb_ops);
1856
1857         qp->digest_idx = digest_idx;
1858
1859         if (processed_jobs < 1)
1860                 processed_jobs += flush_mb_mgr(qp,
1861                                 &ops[processed_jobs],
1862                                 nb_ops - processed_jobs);
1863
1864         return processed_jobs;
1865 }
1866
1867 static MB_MGR *
1868 alloc_init_mb_mgr(enum aesni_mb_vector_mode vector_mode)
1869 {
1870         MB_MGR *mb_mgr = alloc_mb_mgr(0);
1871         if (mb_mgr == NULL)
1872                 return NULL;
1873
1874         switch (vector_mode) {
1875         case RTE_AESNI_MB_SSE:
1876                 init_mb_mgr_sse(mb_mgr);
1877                 break;
1878         case RTE_AESNI_MB_AVX:
1879                 init_mb_mgr_avx(mb_mgr);
1880                 break;
1881         case RTE_AESNI_MB_AVX2:
1882                 init_mb_mgr_avx2(mb_mgr);
1883                 break;
1884         case RTE_AESNI_MB_AVX512:
1885                 init_mb_mgr_avx512(mb_mgr);
1886                 break;
1887         default:
1888                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
1889                 free_mb_mgr(mb_mgr);
1890                 return NULL;
1891         }
1892
1893         return mb_mgr;
1894 }
1895
1896 static inline void
1897 aesni_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
1898 {
1899         uint32_t i;
1900
1901         for (i = 0; i != vec->num; ++i)
1902                 vec->status[i] = err;
1903 }
1904
1905 static inline int
1906 check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)
1907 {
1908         /* no multi-seg support with current AESNI-MB PMD */
1909         if (sgl->num != 1)
1910                 return ENOTSUP;
1911         else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)
1912                 return EINVAL;
1913         return 0;
1914 }
1915
1916 static inline JOB_AES_HMAC *
1917 submit_sync_job(MB_MGR *mb_mgr)
1918 {
1919 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1920         return IMB_SUBMIT_JOB(mb_mgr);
1921 #else
1922         return IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
1923 #endif
1924 }
1925
1926 static inline uint32_t
1927 generate_sync_dgst(struct rte_crypto_sym_vec *vec,
1928         const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1929 {
1930         uint32_t i, k;
1931
1932         for (i = 0, k = 0; i != vec->num; i++) {
1933                 if (vec->status[i] == 0) {
1934                         memcpy(vec->digest[i], dgst[i], len);
1935                         k++;
1936                 }
1937         }
1938
1939         return k;
1940 }
1941
1942 static inline uint32_t
1943 verify_sync_dgst(struct rte_crypto_sym_vec *vec,
1944         const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1945 {
1946         uint32_t i, k;
1947
1948         for (i = 0, k = 0; i != vec->num; i++) {
1949                 if (vec->status[i] == 0) {
1950                         if (memcmp(vec->digest[i], dgst[i], len) != 0)
1951                                 vec->status[i] = EBADMSG;
1952                         else
1953                                 k++;
1954                 }
1955         }
1956
1957         return k;
1958 }
1959
1960 uint32_t
1961 aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
1962         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
1963         struct rte_crypto_sym_vec *vec)
1964 {
1965         int32_t ret;
1966         uint32_t i, j, k, len;
1967         void *buf;
1968         JOB_AES_HMAC *job;
1969         MB_MGR *mb_mgr;
1970         struct aesni_mb_private *priv;
1971         struct aesni_mb_session *s;
1972         uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];
1973
1974         s = get_sym_session_private_data(sess, dev->driver_id);
1975         if (s == NULL) {
1976                 aesni_mb_fill_error_code(vec, EINVAL);
1977                 return 0;
1978         }
1979
1980         /* get per-thread MB MGR, create one if needed */
1981         mb_mgr = RTE_PER_LCORE(sync_mb_mgr);
1982         if (mb_mgr == NULL) {
1983
1984                 priv = dev->data->dev_private;
1985                 mb_mgr = alloc_init_mb_mgr(priv->vector_mode);
1986                 if (mb_mgr == NULL) {
1987                         aesni_mb_fill_error_code(vec, ENOMEM);
1988                         return 0;
1989                 }
1990                 RTE_PER_LCORE(sync_mb_mgr) = mb_mgr;
1991         }
1992
1993         for (i = 0, j = 0, k = 0; i != vec->num; i++) {
1994
1995
1996                 ret = check_crypto_sgl(sofs, vec->sgl + i);
1997                 if (ret != 0) {
1998                         vec->status[i] = ret;
1999                         continue;
2000                 }
2001
2002                 buf = vec->sgl[i].vec[0].base;
2003                 len = vec->sgl[i].vec[0].len;
2004
2005                 job = IMB_GET_NEXT_JOB(mb_mgr);
2006                 if (job == NULL) {
2007                         k += flush_mb_sync_mgr(mb_mgr);
2008                         job = IMB_GET_NEXT_JOB(mb_mgr);
2009                         RTE_ASSERT(job != NULL);
2010                 }
2011
2012                 /* Submit job for processing */
2013                 set_cpu_mb_job_params(job, s, sofs, buf, len,
2014                         vec->iv[i], vec->aad[i], tmp_dgst[i],
2015                         &vec->status[i]);
2016                 job = submit_sync_job(mb_mgr);
2017                 j++;
2018
2019                 /* handle completed jobs */
2020                 k += handle_completed_sync_jobs(job, mb_mgr);
2021         }
2022
2023         /* flush remaining jobs */
2024         while (k != j)
2025                 k += flush_mb_sync_mgr(mb_mgr);
2026
2027         /* finish processing for successful jobs: check/update digest */
2028         if (k != 0) {
2029                 if (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
2030                         k = verify_sync_dgst(vec,
2031                                 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
2032                                 s->auth.req_digest_len);
2033                 else
2034                         k = generate_sync_dgst(vec,
2035                                 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
2036                                 s->auth.req_digest_len);
2037         }
2038
2039         return k;
2040 }
2041
2042 static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
2043
2044 static uint64_t
2045 vec_mode_to_flags(enum aesni_mb_vector_mode mode)
2046 {
2047         switch (mode) {
2048         case RTE_AESNI_MB_SSE:
2049                 return RTE_CRYPTODEV_FF_CPU_SSE;
2050         case RTE_AESNI_MB_AVX:
2051                 return RTE_CRYPTODEV_FF_CPU_AVX;
2052         case RTE_AESNI_MB_AVX2:
2053                 return RTE_CRYPTODEV_FF_CPU_AVX2;
2054         case RTE_AESNI_MB_AVX512:
2055                 return RTE_CRYPTODEV_FF_CPU_AVX512;
2056         default:
2057                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", mode);
2058                 return 0;
2059         }
2060 }
2061
2062 static int
2063 cryptodev_aesni_mb_create(const char *name,
2064                         struct rte_vdev_device *vdev,
2065                         struct rte_cryptodev_pmd_init_params *init_params)
2066 {
2067         struct rte_cryptodev *dev;
2068         struct aesni_mb_private *internals;
2069         enum aesni_mb_vector_mode vector_mode;
2070         MB_MGR *mb_mgr;
2071
2072         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2073         if (dev == NULL) {
2074                 AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
2075                 return -ENODEV;
2076         }
2077
2078         /* Check CPU for supported vector instruction set */
2079         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
2080                 vector_mode = RTE_AESNI_MB_AVX512;
2081         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
2082                 vector_mode = RTE_AESNI_MB_AVX2;
2083         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
2084                 vector_mode = RTE_AESNI_MB_AVX;
2085         else
2086                 vector_mode = RTE_AESNI_MB_SSE;
2087
2088         dev->driver_id = cryptodev_driver_id;
2089         dev->dev_ops = rte_aesni_mb_pmd_ops;
2090
2091         /* register rx/tx burst functions for data path */
2092         dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
2093         dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
2094
2095         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2096                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2097                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
2098                         RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
2099                         RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
2100                         RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
2101
2102 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
2103         struct rte_security_ctx *security_instance;
2104         security_instance = rte_malloc("aesni_mb_sec",
2105                                 sizeof(struct rte_security_ctx),
2106                                 RTE_CACHE_LINE_SIZE);
2107         if (security_instance == NULL) {
2108                 AESNI_MB_LOG(ERR, "rte_security_ctx memory alloc failed");
2109                 rte_cryptodev_pmd_destroy(dev);
2110                 return -ENOMEM;
2111         }
2112
2113         security_instance->device = (void *)dev;
2114         security_instance->ops = rte_aesni_mb_pmd_sec_ops;
2115         security_instance->sess_cnt = 0;
2116         dev->security_ctx = security_instance;
2117         dev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
2118 #endif
2119
2120         /* Check CPU for support for AES instruction set */
2121         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
2122                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
2123         else
2124                 AESNI_MB_LOG(WARNING, "AES instructions not supported by CPU");
2125
2126         dev->feature_flags |= vec_mode_to_flags(vector_mode);
2127
2128         mb_mgr = alloc_init_mb_mgr(vector_mode);
2129         if (mb_mgr == NULL) {
2130 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
2131                 rte_free(dev->security_ctx);
2132                 dev->security_ctx = NULL;
2133 #endif
2134                 rte_cryptodev_pmd_destroy(dev);
2135                 return -ENOMEM;
2136         }
2137
2138         /* Set vector instructions mode supported */
2139         internals = dev->data->dev_private;
2140
2141         internals->vector_mode = vector_mode;
2142         internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
2143         internals->mb_mgr = mb_mgr;
2144
2145         AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
2146                         imb_get_version_str());
2147         return 0;
2148 }
2149
2150 static int
2151 cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
2152 {
2153         struct rte_cryptodev_pmd_init_params init_params = {
2154                 "",
2155                 sizeof(struct aesni_mb_private),
2156                 rte_socket_id(),
2157                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2158         };
2159         const char *name, *args;
2160         int retval;
2161
2162         name = rte_vdev_device_name(vdev);
2163         if (name == NULL)
2164                 return -EINVAL;
2165
2166         args = rte_vdev_device_args(vdev);
2167
2168         retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
2169         if (retval) {
2170                 AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
2171                                 args);
2172                 return -EINVAL;
2173         }
2174
2175         return cryptodev_aesni_mb_create(name, vdev, &init_params);
2176 }
2177
2178 static int
2179 cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
2180 {
2181         struct rte_cryptodev *cryptodev;
2182         struct aesni_mb_private *internals;
2183         const char *name;
2184
2185         name = rte_vdev_device_name(vdev);
2186         if (name == NULL)
2187                 return -EINVAL;
2188
2189         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2190         if (cryptodev == NULL)
2191                 return -ENODEV;
2192
2193         internals = cryptodev->data->dev_private;
2194
2195         free_mb_mgr(internals->mb_mgr);
2196         if (RTE_PER_LCORE(sync_mb_mgr)) {
2197                 free_mb_mgr(RTE_PER_LCORE(sync_mb_mgr));
2198                 RTE_PER_LCORE(sync_mb_mgr) = NULL;
2199         }
2200
2201 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
2202         rte_free(cryptodev->security_ctx);
2203         cryptodev->security_ctx = NULL;
2204 #endif
2205
2206         return rte_cryptodev_pmd_destroy(cryptodev);
2207 }
2208
2209 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
2210         .probe = cryptodev_aesni_mb_probe,
2211         .remove = cryptodev_aesni_mb_remove
2212 };
2213
2214 static struct cryptodev_driver aesni_mb_crypto_drv;
2215
2216 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
2217 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
2218 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
2219         "max_nb_queue_pairs=<int> "
2220         "socket_id=<int>");
2221 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
2222                 cryptodev_aesni_mb_pmd_drv.driver,
2223                 cryptodev_driver_id);
2224 RTE_LOG_REGISTER(aesni_mb_logtype_driver, pmd.crypto.aesni_mb, NOTICE);