drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <intel-ipsec-mb.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_hexdump.h>
   9 #include <rte_cryptodev.h>
  10 #include <rte_cryptodev_pmd.h>
  11 #include <rte_bus_vdev.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cpuflags.h>
  14 #include <rte_per_lcore.h>
  15 #include <rte_ether.h>
  16
  17 #include "aesni_mb_pmd_private.h"
  18
  19 #define AES_CCM_DIGEST_MIN_LEN 4
  20 #define AES_CCM_DIGEST_MAX_LEN 16
  21 #define HMAC_MAX_BLOCK_SIZE 128
  22 static uint8_t cryptodev_driver_id;
  23
  24 /*
  25  * Needed to support CPU-CRYPTO API (rte_cryptodev_sym_cpu_crypto_process),
  26  * as we still use JOB based API even for synchronous processing.
  27  */
  28 static RTE_DEFINE_PER_LCORE(MB_MGR *, sync_mb_mgr);
  29
  30 typedef void (*hash_one_block_t)(const void *data, void *digest);
  31 typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
  32
  33 /**
  34  * Calculate the authentication pre-computes
  35  *
  36  * @param one_block_hash        Function pointer to calculate digest on ipad/opad
  37  * @param ipad                  Inner pad output byte array
  38  * @param opad                  Outer pad output byte array
  39  * @param hkey                  Authentication key
  40  * @param hkey_len              Authentication key length
  41  * @param blocksize             Block size of selected hash algo
  42  */
  43 static void
  44 calculate_auth_precomputes(hash_one_block_t one_block_hash,
  45                 uint8_t *ipad, uint8_t *opad,
  46                 const uint8_t *hkey, uint16_t hkey_len,
  47                 uint16_t blocksize)
  48 {
  49         unsigned i, length;
  50
  51         uint8_t ipad_buf[blocksize] __rte_aligned(16);
  52         uint8_t opad_buf[blocksize] __rte_aligned(16);
  53
  54         /* Setup inner and outer pads */
  55         memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
  56         memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
  57
  58         /* XOR hash key with inner and outer pads */
  59         length = hkey_len > blocksize ? blocksize : hkey_len;
  60
  61         for (i = 0; i < length; i++) {
  62                 ipad_buf[i] ^= hkey[i];
  63                 opad_buf[i] ^= hkey[i];
  64         }
  65
  66         /* Compute partial hashes */
  67         (*one_block_hash)(ipad_buf, ipad);
  68         (*one_block_hash)(opad_buf, opad);
  69
  70         /* Clean up stack */
  71         memset(ipad_buf, 0, blocksize);
  72         memset(opad_buf, 0, blocksize);
  73 }
  74
  75 /** Get xform chain order */
  76 static enum aesni_mb_operation
  77 aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
  78 {
  79         if (xform == NULL)
  80                 return AESNI_MB_OP_NOT_SUPPORTED;
  81
  82         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  83                 if (xform->next == NULL)
  84                         return AESNI_MB_OP_CIPHER_ONLY;
  85                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  86                         return AESNI_MB_OP_CIPHER_HASH;
  87         }
  88
  89         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  90                 if (xform->next == NULL)
  91                         return AESNI_MB_OP_HASH_ONLY;
  92                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
  93                         return AESNI_MB_OP_HASH_CIPHER;
  94         }
  95 #if IMB_VERSION_NUM > IMB_VERSION(0, 52, 0)
  96         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  97                 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
  98                         /*
  99                          * CCM requires to hash first and cipher later
 100                          * when encrypting
 101                          */
 102                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 103                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 104                         else
 105                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 106                 } else {
 107                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 108                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 109                         else
 110                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 111                 }
 112         }
 113 #else
 114         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 115                 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
 116                                 xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
 117                         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 118                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 119                         else
 120                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 121                 }
 122         }
 123 #endif
 124
 125         return AESNI_MB_OP_NOT_SUPPORTED;
 126 }
 127
 128 /** Set session authentication parameters */
 129 static int
 130 aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
 131                 struct aesni_mb_session *sess,
 132                 const struct rte_crypto_sym_xform *xform)
 133 {
 134         hash_one_block_t hash_oneblock_fn = NULL;
 135         unsigned int key_larger_block_size = 0;
 136         uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
 137         uint32_t auth_precompute = 1;
 138
 139         if (xform == NULL) {
 140                 sess->auth.algo = NULL_HASH;
 141                 return 0;
 142         }
 143
 144         if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
 145                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
 146                 return -1;
 147         }
 148
 149         /* Set the request digest size */
 150         sess->auth.req_digest_len = xform->auth.digest_length;
 151
 152         /* Select auth generate/verify */
 153         sess->auth.operation = xform->auth.op;
 154
 155         /* Set Authentication Parameters */
 156         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
 157                 sess->auth.algo = AES_XCBC;
 158
 159                 uint16_t xcbc_mac_digest_len =
 160                         get_truncated_digest_byte_length(AES_XCBC);
 161                 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
 162                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 163                         return -EINVAL;
 164                 }
 165                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 166
 167                 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
 168                                 sess->auth.xcbc.k1_expanded,
 169                                 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
 170                 return 0;
 171         }
 172
 173         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
 174                 uint32_t dust[4*15];
 175
 176                 sess->auth.algo = AES_CMAC;
 177
 178                 uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
 179
 180                 if (sess->auth.req_digest_len > cmac_digest_len) {
 181                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 182                         return -EINVAL;
 183                 }
 184                 /*
 185                  * Multi-buffer lib supports digest sizes from 4 to 16 bytes
 186                  * in version 0.50 and sizes of 12 and 16 bytes,
 187                  * in version 0.49.
 188                  * If size requested is different, generate the full digest
 189                  * (16 bytes) in a temporary location and then memcpy
 190                  * the requested number of bytes.
 191                  */
 192                 if (sess->auth.req_digest_len < 4)
 193                         sess->auth.gen_digest_len = cmac_digest_len;
 194                 else
 195                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 196
 197                 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
 198                                 sess->auth.cmac.expkey, dust);
 199                 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
 200                                 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
 201                 return 0;
 202         }
 203
 204         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 205                 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 206                         sess->cipher.direction = ENCRYPT;
 207                         sess->chain_order = CIPHER_HASH;
 208                 } else
 209                         sess->cipher.direction = DECRYPT;
 210
 211                 sess->auth.algo = AES_GMAC;
 212                 /*
 213                  * Multi-buffer lib supports 8, 12 and 16 bytes of digest.
 214                  * If size requested is different, generate the full digest
 215                  * (16 bytes) in a temporary location and then memcpy
 216                  * the requested number of bytes.
 217                  */
 218                 if (sess->auth.req_digest_len != 16 &&
 219                                 sess->auth.req_digest_len != 12 &&
 220                                 sess->auth.req_digest_len != 8) {
 221                         sess->auth.gen_digest_len = 16;
 222                 } else {
 223                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 224                 }
 225                 sess->iv.length = xform->auth.iv.length;
 226                 sess->iv.offset = xform->auth.iv.offset;
 227
 228                 switch (xform->auth.key.length) {
 229                 case AES_128_BYTES:
 230                         IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
 231                                 &sess->cipher.gcm_key);
 232                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 233                         break;
 234                 case AES_192_BYTES:
 235                         IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
 236                                 &sess->cipher.gcm_key);
 237                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 238                         break;
 239                 case AES_256_BYTES:
 240                         IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
 241                                 &sess->cipher.gcm_key);
 242                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 243                         break;
 244                 default:
 245                         RTE_LOG(ERR, PMD, "failed to parse test type\n");
 246                         return -EINVAL;
 247                 }
 248
 249                 return 0;
 250         }
 251
 252         switch (xform->auth.algo) {
 253         case RTE_CRYPTO_AUTH_MD5_HMAC:
 254                 sess->auth.algo = MD5;
 255                 hash_oneblock_fn = mb_mgr->md5_one_block;
 256                 break;
 257         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 258                 sess->auth.algo = SHA1;
 259                 hash_oneblock_fn = mb_mgr->sha1_one_block;
 260                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
 261                         IMB_SHA1(mb_mgr,
 262                                 xform->auth.key.data,
 263                                 xform->auth.key.length,
 264                                 hashed_key);
 265                         key_larger_block_size = 1;
 266                 }
 267                 break;
 268         case RTE_CRYPTO_AUTH_SHA1:
 269                 sess->auth.algo = PLAIN_SHA1;
 270                 auth_precompute = 0;
 271                 break;
 272         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 273                 sess->auth.algo = SHA_224;
 274                 hash_oneblock_fn = mb_mgr->sha224_one_block;
 275                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
 276                         IMB_SHA224(mb_mgr,
 277                                 xform->auth.key.data,
 278                                 xform->auth.key.length,
 279                                 hashed_key);
 280                         key_larger_block_size = 1;
 281                 }
 282                 break;
 283         case RTE_CRYPTO_AUTH_SHA224:
 284                 sess->auth.algo = PLAIN_SHA_224;
 285                 auth_precompute = 0;
 286                 break;
 287         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 288                 sess->auth.algo = SHA_256;
 289                 hash_oneblock_fn = mb_mgr->sha256_one_block;
 290                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
 291                         IMB_SHA256(mb_mgr,
 292                                 xform->auth.key.data,
 293                                 xform->auth.key.length,
 294                                 hashed_key);
 295                         key_larger_block_size = 1;
 296                 }
 297                 break;
 298         case RTE_CRYPTO_AUTH_SHA256:
 299                 sess->auth.algo = PLAIN_SHA_256;
 300                 auth_precompute = 0;
 301                 break;
 302         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 303                 sess->auth.algo = SHA_384;
 304                 hash_oneblock_fn = mb_mgr->sha384_one_block;
 305                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
 306                         IMB_SHA384(mb_mgr,
 307                                 xform->auth.key.data,
 308                                 xform->auth.key.length,
 309                                 hashed_key);
 310                         key_larger_block_size = 1;
 311                 }
 312                 break;
 313         case RTE_CRYPTO_AUTH_SHA384:
 314                 sess->auth.algo = PLAIN_SHA_384;
 315                 auth_precompute = 0;
 316                 break;
 317         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 318                 sess->auth.algo = SHA_512;
 319                 hash_oneblock_fn = mb_mgr->sha512_one_block;
 320                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
 321                         IMB_SHA512(mb_mgr,
 322                                 xform->auth.key.data,
 323                                 xform->auth.key.length,
 324                                 hashed_key);
 325                         key_larger_block_size = 1;
 326                 }
 327                 break;
 328         case RTE_CRYPTO_AUTH_SHA512:
 329                 sess->auth.algo = PLAIN_SHA_512;
 330                 auth_precompute = 0;
 331                 break;
 332         default:
 333                 AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
 334                 return -ENOTSUP;
 335         }
 336         uint16_t trunc_digest_size =
 337                         get_truncated_digest_byte_length(sess->auth.algo);
 338         uint16_t full_digest_size =
 339                         get_digest_byte_length(sess->auth.algo);
 340
 341         if (sess->auth.req_digest_len > full_digest_size ||
 342                         sess->auth.req_digest_len == 0) {
 343                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 344                 return -EINVAL;
 345         }
 346
 347         if (sess->auth.req_digest_len != trunc_digest_size &&
 348                         sess->auth.req_digest_len != full_digest_size)
 349                 sess->auth.gen_digest_len = full_digest_size;
 350         else
 351                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 352
 353         /* Plain SHA does not require precompute key */
 354         if (auth_precompute == 0)
 355                 return 0;
 356
 357         /* Calculate Authentication precomputes */
 358         if (key_larger_block_size) {
 359                 calculate_auth_precomputes(hash_oneblock_fn,
 360                         sess->auth.pads.inner, sess->auth.pads.outer,
 361                         hashed_key,
 362                         xform->auth.key.length,
 363                         get_auth_algo_blocksize(sess->auth.algo));
 364         } else {
 365                 calculate_auth_precomputes(hash_oneblock_fn,
 366                         sess->auth.pads.inner, sess->auth.pads.outer,
 367                         xform->auth.key.data,
 368                         xform->auth.key.length,
 369                         get_auth_algo_blocksize(sess->auth.algo));
 370         }
 371
 372         return 0;
 373 }
 374
 375 /** Set session cipher parameters */
 376 static int
 377 aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
 378                 struct aesni_mb_session *sess,
 379                 const struct rte_crypto_sym_xform *xform)
 380 {
 381         uint8_t is_aes = 0;
 382         uint8_t is_3DES = 0;
 383         uint8_t is_docsis = 0;
 384
 385         if (xform == NULL) {
 386                 sess->cipher.mode = NULL_CIPHER;
 387                 return 0;
 388         }
 389
 390         if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
 391                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
 392                 return -EINVAL;
 393         }
 394
 395         /* Select cipher direction */
 396         switch (xform->cipher.op) {
 397         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
 398                 sess->cipher.direction = ENCRYPT;
 399                 break;
 400         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
 401                 sess->cipher.direction = DECRYPT;
 402                 break;
 403         default:
 404                 AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
 405                 return -EINVAL;
 406         }
 407
 408         /* Select cipher mode */
 409         switch (xform->cipher.algo) {
 410         case RTE_CRYPTO_CIPHER_AES_CBC:
 411                 sess->cipher.mode = CBC;
 412                 is_aes = 1;
 413                 break;
 414         case RTE_CRYPTO_CIPHER_AES_CTR:
 415                 sess->cipher.mode = CNTR;
 416                 is_aes = 1;
 417                 break;
 418         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 419                 sess->cipher.mode = DOCSIS_SEC_BPI;
 420                 is_docsis = 1;
 421                 break;
 422         case RTE_CRYPTO_CIPHER_DES_CBC:
 423                 sess->cipher.mode = DES;
 424                 break;
 425         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 426                 sess->cipher.mode = DOCSIS_DES;
 427                 break;
 428         case RTE_CRYPTO_CIPHER_3DES_CBC:
 429                 sess->cipher.mode = DES3;
 430                 is_3DES = 1;
 431                 break;
 432         default:
 433                 AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
 434                 return -ENOTSUP;
 435         }
 436
 437         /* Set IV parameters */
 438         sess->iv.offset = xform->cipher.iv.offset;
 439         sess->iv.length = xform->cipher.iv.length;
 440
 441         /* Check key length and choose key expansion function for AES */
 442         if (is_aes) {
 443                 switch (xform->cipher.key.length) {
 444                 case AES_128_BYTES:
 445                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 446                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 447                                         sess->cipher.expanded_aes_keys.encode,
 448                                         sess->cipher.expanded_aes_keys.decode);
 449                         break;
 450                 case AES_192_BYTES:
 451                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 452                         IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
 453                                         sess->cipher.expanded_aes_keys.encode,
 454                                         sess->cipher.expanded_aes_keys.decode);
 455                         break;
 456                 case AES_256_BYTES:
 457                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 458                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 459                                         sess->cipher.expanded_aes_keys.encode,
 460                                         sess->cipher.expanded_aes_keys.decode);
 461                         break;
 462                 default:
 463                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 464                         return -EINVAL;
 465                 }
 466         } else if (is_docsis) {
 467                 switch (xform->cipher.key.length) {
 468                 case AES_128_BYTES:
 469                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 470                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 471                                         sess->cipher.expanded_aes_keys.encode,
 472                                         sess->cipher.expanded_aes_keys.decode);
 473                         break;
 474 #if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
 475                 case AES_256_BYTES:
 476                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 477                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 478                                         sess->cipher.expanded_aes_keys.encode,
 479                                         sess->cipher.expanded_aes_keys.decode);
 480                         break;
 481 #endif
 482                 default:
 483                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 484                         return -EINVAL;
 485                 }
 486         } else if (is_3DES) {
 487                 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
 488                                 sess->cipher.exp_3des_keys.key[1],
 489                                 sess->cipher.exp_3des_keys.key[2]};
 490
 491                 switch (xform->cipher.key.length) {
 492                 case  24:
 493                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 494                                         xform->cipher.key.data);
 495                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 496                                         xform->cipher.key.data + 8);
 497                         IMB_DES_KEYSCHED(mb_mgr, keys[2],
 498                                         xform->cipher.key.data + 16);
 499
 500                         /* Initialize keys - 24 bytes: [K1-K2-K3] */
 501                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 502                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 503                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
 504                         break;
 505                 case 16:
 506                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 507                                         xform->cipher.key.data);
 508                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 509                                         xform->cipher.key.data + 8);
 510                         /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
 511                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 512                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 513                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 514                         break;
 515                 case 8:
 516                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 517                                         xform->cipher.key.data);
 518
 519                         /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
 520                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 521                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
 522                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 523                         break;
 524                 default:
 525                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 526                         return -EINVAL;
 527                 }
 528
 529                 sess->cipher.key_length_in_bytes = 24;
 530         } else {
 531                 if (xform->cipher.key.length != 8) {
 532                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 533                         return -EINVAL;
 534                 }
 535                 sess->cipher.key_length_in_bytes = 8;
 536
 537                 IMB_DES_KEYSCHED(mb_mgr,
 538                         (uint64_t *)sess->cipher.expanded_aes_keys.encode,
 539                                 xform->cipher.key.data);
 540                 IMB_DES_KEYSCHED(mb_mgr,
 541                         (uint64_t *)sess->cipher.expanded_aes_keys.decode,
 542                                 xform->cipher.key.data);
 543         }
 544
 545         return 0;
 546 }
 547
 548 static int
 549 aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
 550                 struct aesni_mb_session *sess,
 551                 const struct rte_crypto_sym_xform *xform)
 552 {
 553         switch (xform->aead.op) {
 554         case RTE_CRYPTO_AEAD_OP_ENCRYPT:
 555                 sess->cipher.direction = ENCRYPT;
 556                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 557                 break;
 558         case RTE_CRYPTO_AEAD_OP_DECRYPT:
 559                 sess->cipher.direction = DECRYPT;
 560                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 561                 break;
 562         default:
 563                 AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
 564                 return -EINVAL;
 565         }
 566
 567         switch (xform->aead.algo) {
 568         case RTE_CRYPTO_AEAD_AES_CCM:
 569                 sess->cipher.mode = CCM;
 570                 sess->auth.algo = AES_CCM;
 571
 572                 /* Check key length and choose key expansion function for AES */
 573                 switch (xform->aead.key.length) {
 574                 case AES_128_BYTES:
 575                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 576                         IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
 577                                         sess->cipher.expanded_aes_keys.encode,
 578                                         sess->cipher.expanded_aes_keys.decode);
 579                         break;
 580                 default:
 581                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 582                         return -EINVAL;
 583                 }
 584
 585                 break;
 586
 587         case RTE_CRYPTO_AEAD_AES_GCM:
 588                 sess->cipher.mode = GCM;
 589                 sess->auth.algo = AES_GMAC;
 590
 591                 switch (xform->aead.key.length) {
 592                 case AES_128_BYTES:
 593                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 594                         IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
 595                                 &sess->cipher.gcm_key);
 596                         break;
 597                 case AES_192_BYTES:
 598                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 599                         IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
 600                                 &sess->cipher.gcm_key);
 601                         break;
 602                 case AES_256_BYTES:
 603                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 604                         IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
 605                                 &sess->cipher.gcm_key);
 606                         break;
 607                 default:
 608                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 609                         return -EINVAL;
 610                 }
 611
 612                 break;
 613
 614         default:
 615                 AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
 616                 return -ENOTSUP;
 617         }
 618
 619         /* Set IV parameters */
 620         sess->iv.offset = xform->aead.iv.offset;
 621         sess->iv.length = xform->aead.iv.length;
 622
 623         sess->auth.req_digest_len = xform->aead.digest_length;
 624         /* CCM digests must be between 4 and 16 and an even number */
 625         if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
 626                         sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
 627                         (sess->auth.req_digest_len & 1) == 1) {
 628                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 629                 return -EINVAL;
 630         }
 631         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 632
 633         return 0;
 634 }
 635
 636 /** Parse crypto xform chain and set private session parameters */
 637 int
 638 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 639                 struct aesni_mb_session *sess,
 640                 const struct rte_crypto_sym_xform *xform)
 641 {
 642         const struct rte_crypto_sym_xform *auth_xform = NULL;
 643         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 644         const struct rte_crypto_sym_xform *aead_xform = NULL;
 645         int ret;
 646
 647         /* Select Crypto operation - hash then cipher / cipher then hash */
 648         switch (aesni_mb_get_chain_order(xform)) {
 649         case AESNI_MB_OP_HASH_CIPHER:
 650                 sess->chain_order = HASH_CIPHER;
 651                 auth_xform = xform;
 652                 cipher_xform = xform->next;
 653                 break;
 654         case AESNI_MB_OP_CIPHER_HASH:
 655                 sess->chain_order = CIPHER_HASH;
 656                 auth_xform = xform->next;
 657                 cipher_xform = xform;
 658                 break;
 659         case AESNI_MB_OP_HASH_ONLY:
 660                 sess->chain_order = HASH_CIPHER;
 661                 auth_xform = xform;
 662                 cipher_xform = NULL;
 663                 break;
 664         case AESNI_MB_OP_CIPHER_ONLY:
 665                 /*
 666                  * Multi buffer library operates only at two modes,
 667                  * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
 668                  * chain order depends on cipher operation: encryption is always
 669                  * the first operation and decryption the last one.
 670                  */
 671                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 672                         sess->chain_order = CIPHER_HASH;
 673                 else
 674                         sess->chain_order = HASH_CIPHER;
 675                 auth_xform = NULL;
 676                 cipher_xform = xform;
 677                 break;
 678         case AESNI_MB_OP_AEAD_CIPHER_HASH:
 679                 sess->chain_order = CIPHER_HASH;
 680                 sess->aead.aad_len = xform->aead.aad_length;
 681                 aead_xform = xform;
 682                 break;
 683         case AESNI_MB_OP_AEAD_HASH_CIPHER:
 684                 sess->chain_order = HASH_CIPHER;
 685                 sess->aead.aad_len = xform->aead.aad_length;
 686                 aead_xform = xform;
 687                 break;
 688         case AESNI_MB_OP_NOT_SUPPORTED:
 689         default:
 690                 AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
 691                 return -ENOTSUP;
 692         }
 693
 694         /* Default IV length = 0 */
 695         sess->iv.length = 0;
 696
 697         ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
 698         if (ret != 0) {
 699                 AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
 700                 return ret;
 701         }
 702
 703         ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
 704                         cipher_xform);
 705         if (ret != 0) {
 706                 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 707                 return ret;
 708         }
 709
 710         if (aead_xform) {
 711                 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
 712                                 aead_xform);
 713                 if (ret != 0) {
 714                         AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
 715                         return ret;
 716                 }
 717         }
 718
 719         return 0;
 720 }
 721
 722 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 723 /** Check DOCSIS security session configuration is valid */
 724 static int
 725 check_docsis_sec_session(struct rte_security_session_conf *conf)
 726 {
 727         struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;
 728         struct rte_security_docsis_xform *docsis = &conf->docsis;
 729
 730         /* Downlink: CRC generate -> Cipher encrypt */
 731         if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
 732
 733                 if (crypto_sym != NULL &&
 734                     crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 735                     crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
 736                     crypto_sym->cipher.algo ==
 737                                         RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
 738                     (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||
 739                      crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&
 740                     crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
 741                     crypto_sym->next == NULL) {
 742                         return 0;
 743                 }
 744         /* Uplink: Cipher decrypt -> CRC verify */
 745         } else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {
 746
 747                 if (crypto_sym != NULL &&
 748                     crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 749                     crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
 750                     crypto_sym->cipher.algo ==
 751                                         RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
 752                     (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES ||
 753                      crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) &&
 754                     crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
 755                     crypto_sym->next == NULL) {
 756                         return 0;
 757                 }
 758         }
 759
 760         return -EINVAL;
 761 }
 762
 763 /** Set DOCSIS security session auth (CRC) parameters */
 764 static int
 765 aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
 766                 struct rte_security_docsis_xform *xform)
 767 {
 768         if (xform == NULL) {
 769                 AESNI_MB_LOG(ERR, "Invalid DOCSIS xform");
 770                 return -EINVAL;
 771         }
 772
 773         /* Select CRC generate/verify */
 774         if (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) {
 775                 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
 776                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 777         } else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
 778                 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
 779                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 780         } else {
 781                 AESNI_MB_LOG(ERR, "Unsupported DOCSIS direction");
 782                 return -ENOTSUP;
 783         }
 784
 785         sess->auth.req_digest_len = RTE_ETHER_CRC_LEN;
 786         sess->auth.gen_digest_len = RTE_ETHER_CRC_LEN;
 787
 788         return 0;
 789 }
 790
 791 /**
 792  * Parse DOCSIS security session configuration and set private session
 793  * parameters
 794  */
 795 int
 796 aesni_mb_set_docsis_sec_session_parameters(
 797                 __rte_unused struct rte_cryptodev *dev,
 798                 struct rte_security_session_conf *conf,
 799                 void *sess)
 800 {
 801         struct rte_security_docsis_xform *docsis_xform;
 802         struct rte_crypto_sym_xform *cipher_xform;
 803         struct aesni_mb_session *aesni_sess = sess;
 804         struct aesni_mb_private *internals = dev->data->dev_private;
 805         int ret;
 806
 807         ret = check_docsis_sec_session(conf);
 808         if (ret) {
 809                 AESNI_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
 810                 return ret;
 811         }
 812
 813         switch (conf->docsis.direction) {
 814         case RTE_SECURITY_DOCSIS_UPLINK:
 815                 aesni_sess->chain_order = IMB_ORDER_CIPHER_HASH;
 816                 docsis_xform = &conf->docsis;
 817                 cipher_xform = conf->crypto_xform;
 818                 break;
 819         case RTE_SECURITY_DOCSIS_DOWNLINK:
 820                 aesni_sess->chain_order = IMB_ORDER_HASH_CIPHER;
 821                 cipher_xform = conf->crypto_xform;
 822                 docsis_xform = &conf->docsis;
 823                 break;
 824         default:
 825                 return -EINVAL;
 826         }
 827
 828         /* Default IV length = 0 */
 829         aesni_sess->iv.length = 0;
 830
 831         ret = aesni_mb_set_docsis_sec_session_auth_parameters(aesni_sess,
 832                         docsis_xform);
 833         if (ret != 0) {
 834                 AESNI_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
 835                 return -EINVAL;
 836         }
 837
 838         ret = aesni_mb_set_session_cipher_parameters(internals->mb_mgr,
 839                         aesni_sess, cipher_xform);
 840
 841         if (ret != 0) {
 842                 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 843                 return -EINVAL;
 844         }
 845
 846         return 0;
 847 }
 848 #endif
 849
 850 /**
 851  * burst enqueue, place crypto operations on ingress queue for processing.
 852  *
 853  * @param __qp         Queue Pair to process
 854  * @param ops          Crypto operations for processing
 855  * @param nb_ops       Number of crypto operations for processing
 856  *
 857  * @return
 858  * - Number of crypto operations enqueued
 859  */
 860 static uint16_t
 861 aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
 862                 uint16_t nb_ops)
 863 {
 864         struct aesni_mb_qp *qp = __qp;
 865
 866         unsigned int nb_enqueued;
 867
 868         nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
 869                         (void **)ops, nb_ops, NULL);
 870
 871         qp->stats.enqueued_count += nb_enqueued;
 872
 873         return nb_enqueued;
 874 }
 875
 876 /** Get multi buffer session */
 877 static inline struct aesni_mb_session *
 878 get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
 879 {
 880         struct aesni_mb_session *sess = NULL;
 881
 882         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 883                 if (likely(op->sym->session != NULL))
 884                         sess = (struct aesni_mb_session *)
 885                                         get_sym_session_private_data(
 886                                         op->sym->session,
 887                                         cryptodev_driver_id);
 888 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 889         } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 890                 if (likely(op->sym->sec_session != NULL))
 891                         sess = (struct aesni_mb_session *)
 892                                         get_sec_session_private_data(
 893                                                 op->sym->sec_session);
 894 #endif
 895         } else {
 896                 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
 897                 void *_sess_private_data = NULL;
 898
 899                 if (_sess == NULL)
 900                         return NULL;
 901
 902                 if (rte_mempool_get(qp->sess_mp_priv,
 903                                 (void **)&_sess_private_data))
 904                         return NULL;
 905
 906                 sess = (struct aesni_mb_session *)_sess_private_data;
 907
 908                 if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
 909                                 sess, op->sym->xform) != 0)) {
 910                         rte_mempool_put(qp->sess_mp, _sess);
 911                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 912                         sess = NULL;
 913                 }
 914                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 915                 set_sym_session_private_data(op->sym->session,
 916                                 cryptodev_driver_id, _sess_private_data);
 917         }
 918
 919         if (unlikely(sess == NULL))
 920                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 921
 922         return sess;
 923 }
 924
 925 static inline uint64_t
 926 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 927                 uint32_t oop)
 928 {
 929         struct rte_mbuf *m_src, *m_dst;
 930         uint8_t *p_src, *p_dst;
 931         uintptr_t u_src, u_dst;
 932         uint32_t cipher_end, auth_end;
 933
 934         /* Only cipher then hash needs special calculation. */
 935         if (!oop || session->chain_order != CIPHER_HASH)
 936                 return op->sym->auth.data.offset;
 937
 938         m_src = op->sym->m_src;
 939         m_dst = op->sym->m_dst;
 940
 941         p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
 942         p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
 943         u_src = (uintptr_t)p_src;
 944         u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
 945
 946         /**
 947          * Copy the content between cipher offset and auth offset for generating
 948          * correct digest.
 949          */
 950         if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
 951                 memcpy(p_dst + op->sym->auth.data.offset,
 952                                 p_src + op->sym->auth.data.offset,
 953                                 op->sym->cipher.data.offset -
 954                                 op->sym->auth.data.offset);
 955
 956         /**
 957          * Copy the content between (cipher offset + length) and (auth offset +
 958          * length) for generating correct digest
 959          */
 960         cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
 961         auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
 962         if (cipher_end < auth_end)
 963                 memcpy(p_dst + cipher_end, p_src + cipher_end,
 964                                 auth_end - cipher_end);
 965
 966         /**
 967          * Since intel-ipsec-mb only supports positive values,
 968          * we need to deduct the correct offset between src and dst.
 969          */
 970
 971         return u_src < u_dst ? (u_dst - u_src) :
 972                         (UINT64_MAX - u_src + u_dst + 1);
 973 }
 974
 975 static inline void
 976 set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
 977                 union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
 978                 void *iv, void *aad, void *digest, void *udata)
 979 {
 980         /* Set crypto operation */
 981         job->chain_order = session->chain_order;
 982
 983         /* Set cipher parameters */
 984         job->cipher_direction = session->cipher.direction;
 985         job->cipher_mode = session->cipher.mode;
 986
 987         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
 988
 989         /* Set authentication parameters */
 990         job->hash_alg = session->auth.algo;
 991         job->iv = iv;
 992
 993         switch (job->hash_alg) {
 994         case AES_XCBC:
 995                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
 996                 job->u.XCBC._k2 = session->auth.xcbc.k2;
 997                 job->u.XCBC._k3 = session->auth.xcbc.k3;
 998
 999                 job->aes_enc_key_expanded =
1000                                 session->cipher.expanded_aes_keys.encode;
1001                 job->aes_dec_key_expanded =
1002                                 session->cipher.expanded_aes_keys.decode;
1003                 break;
1004
1005         case AES_CCM:
1006                 job->u.CCM.aad = (uint8_t *)aad + 18;
1007                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1008                 job->aes_enc_key_expanded =
1009                                 session->cipher.expanded_aes_keys.encode;
1010                 job->aes_dec_key_expanded =
1011                                 session->cipher.expanded_aes_keys.decode;
1012                 job->iv++;
1013                 break;
1014
1015         case AES_CMAC:
1016                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1017                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1018                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1019                 job->aes_enc_key_expanded =
1020                                 session->cipher.expanded_aes_keys.encode;
1021                 job->aes_dec_key_expanded =
1022                                 session->cipher.expanded_aes_keys.decode;
1023                 break;
1024
1025         case AES_GMAC:
1026                 if (session->cipher.mode == GCM) {
1027                         job->u.GCM.aad = aad;
1028                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1029                 } else {
1030                         /* For GMAC */
1031                         job->u.GCM.aad = buf;
1032                         job->u.GCM.aad_len_in_bytes = len;
1033                         job->cipher_mode = GCM;
1034                 }
1035                 job->aes_enc_key_expanded = &session->cipher.gcm_key;
1036                 job->aes_dec_key_expanded = &session->cipher.gcm_key;
1037                 break;
1038
1039         default:
1040                 job->u.HMAC._hashed_auth_key_xor_ipad =
1041                                 session->auth.pads.inner;
1042                 job->u.HMAC._hashed_auth_key_xor_opad =
1043                                 session->auth.pads.outer;
1044
1045                 if (job->cipher_mode == DES3) {
1046                         job->aes_enc_key_expanded =
1047                                 session->cipher.exp_3des_keys.ks_ptr;
1048                         job->aes_dec_key_expanded =
1049                                 session->cipher.exp_3des_keys.ks_ptr;
1050                 } else {
1051                         job->aes_enc_key_expanded =
1052                                 session->cipher.expanded_aes_keys.encode;
1053                         job->aes_dec_key_expanded =
1054                                 session->cipher.expanded_aes_keys.decode;
1055                 }
1056         }
1057
1058         /*
1059          * Multi-buffer library current only support returning a truncated
1060          * digest length as specified in the relevant IPsec RFCs
1061          */
1062
1063         /* Set digest location and length */
1064         job->auth_tag_output = digest;
1065         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1066
1067         /* Set IV parameters */
1068         job->iv_len_in_bytes = session->iv.length;
1069
1070         /* Data Parameters */
1071         job->src = buf;
1072         job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;
1073         job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;
1074         job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;
1075         if (job->hash_alg == AES_GMAC && session->cipher.mode != GCM) {
1076                 job->msg_len_to_hash_in_bytes = 0;
1077                 job->msg_len_to_cipher_in_bytes = 0;
1078         } else {
1079                 job->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -
1080                         sofs.ofs.auth.tail;
1081                 job->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -
1082                         sofs.ofs.cipher.tail;
1083         }
1084
1085         job->user_data = udata;
1086 }
1087
1088 /**
1089  * Process a crypto operation and complete a JOB_AES_HMAC job structure for
1090  * submission to the multi buffer library for processing.
1091  *
1092  * @param       qp      queue pair
1093  * @param       job     JOB_AES_HMAC structure to fill
1094  * @param       m       mbuf to process
1095  *
1096  * @return
1097  * - Completed JOB_AES_HMAC structure pointer on success
1098  * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
1099  */
1100 static inline int
1101 set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
1102                 struct rte_crypto_op *op, uint8_t *digest_idx)
1103 {
1104         struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
1105         struct aesni_mb_session *session;
1106         uint32_t m_offset, oop;
1107
1108         session = get_session(qp, op);
1109         if (session == NULL) {
1110                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1111                 return -1;
1112         }
1113
1114         /* Set crypto operation */
1115         job->chain_order = session->chain_order;
1116
1117         /* Set cipher parameters */
1118         job->cipher_direction = session->cipher.direction;
1119         job->cipher_mode = session->cipher.mode;
1120
1121         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
1122
1123         /* Set authentication parameters */
1124         job->hash_alg = session->auth.algo;
1125
1126         switch (job->hash_alg) {
1127         case AES_XCBC:
1128                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
1129                 job->u.XCBC._k2 = session->auth.xcbc.k2;
1130                 job->u.XCBC._k3 = session->auth.xcbc.k3;
1131
1132                 job->aes_enc_key_expanded =
1133                                 session->cipher.expanded_aes_keys.encode;
1134                 job->aes_dec_key_expanded =
1135                                 session->cipher.expanded_aes_keys.decode;
1136                 break;
1137
1138         case AES_CCM:
1139                 job->u.CCM.aad = op->sym->aead.aad.data + 18;
1140                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1141                 job->aes_enc_key_expanded =
1142                                 session->cipher.expanded_aes_keys.encode;
1143                 job->aes_dec_key_expanded =
1144                                 session->cipher.expanded_aes_keys.decode;
1145                 break;
1146
1147         case AES_CMAC:
1148                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1149                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1150                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1151                 job->aes_enc_key_expanded =
1152                                 session->cipher.expanded_aes_keys.encode;
1153                 job->aes_dec_key_expanded =
1154                                 session->cipher.expanded_aes_keys.decode;
1155                 break;
1156
1157         case AES_GMAC:
1158                 if (session->cipher.mode == GCM) {
1159                         job->u.GCM.aad = op->sym->aead.aad.data;
1160                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1161                 } else {
1162                         /* For GMAC */
1163                         job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
1164                                         uint8_t *, op->sym->auth.data.offset);
1165                         job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
1166                         job->cipher_mode = GCM;
1167                 }
1168                 job->aes_enc_key_expanded = &session->cipher.gcm_key;
1169                 job->aes_dec_key_expanded = &session->cipher.gcm_key;
1170                 break;
1171
1172         default:
1173                 job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
1174                 job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
1175
1176                 if (job->cipher_mode == DES3) {
1177                         job->aes_enc_key_expanded =
1178                                 session->cipher.exp_3des_keys.ks_ptr;
1179                         job->aes_dec_key_expanded =
1180                                 session->cipher.exp_3des_keys.ks_ptr;
1181                 } else {
1182                         job->aes_enc_key_expanded =
1183                                 session->cipher.expanded_aes_keys.encode;
1184                         job->aes_dec_key_expanded =
1185                                 session->cipher.expanded_aes_keys.decode;
1186                 }
1187         }
1188
1189         if (!op->sym->m_dst) {
1190                 /* in-place operation */
1191                 m_dst = m_src;
1192                 oop = 0;
1193         } else if (op->sym->m_dst == op->sym->m_src) {
1194                 /* in-place operation */
1195                 m_dst = m_src;
1196                 oop = 0;
1197         } else {
1198                 /* out-of-place operation */
1199                 m_dst = op->sym->m_dst;
1200                 oop = 1;
1201         }
1202
1203         if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
1204                         session->cipher.mode == GCM))
1205                 m_offset = op->sym->aead.data.offset;
1206         else
1207                 m_offset = op->sym->cipher.data.offset;
1208
1209         /* Set digest output location */
1210         if (job->hash_alg != NULL_HASH &&
1211                         session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1212                 job->auth_tag_output = qp->temp_digests[*digest_idx];
1213                 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1214         } else {
1215                 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
1216                                 session->cipher.mode == GCM))
1217                         job->auth_tag_output = op->sym->aead.digest.data;
1218                 else
1219                         job->auth_tag_output = op->sym->auth.digest.data;
1220
1221                 if (session->auth.req_digest_len != session->auth.gen_digest_len) {
1222                         job->auth_tag_output = qp->temp_digests[*digest_idx];
1223                         *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1224                 }
1225         }
1226         /*
1227          * Multi-buffer library current only support returning a truncated
1228          * digest length as specified in the relevant IPsec RFCs
1229          */
1230
1231         /* Set digest length */
1232         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1233
1234         /* Set IV parameters */
1235         job->iv_len_in_bytes = session->iv.length;
1236
1237         /* Data Parameters */
1238         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1239         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
1240
1241         switch (job->hash_alg) {
1242         case AES_CCM:
1243                 job->cipher_start_src_offset_in_bytes =
1244                                 op->sym->aead.data.offset;
1245                 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
1246                 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
1247                 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
1248
1249                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1250                         session->iv.offset + 1);
1251                 break;
1252
1253         case AES_GMAC:
1254                 if (session->cipher.mode == GCM) {
1255                         job->cipher_start_src_offset_in_bytes =
1256                                         op->sym->aead.data.offset;
1257                         job->hash_start_src_offset_in_bytes =
1258                                         op->sym->aead.data.offset;
1259                         job->msg_len_to_cipher_in_bytes =
1260                                         op->sym->aead.data.length;
1261                         job->msg_len_to_hash_in_bytes =
1262                                         op->sym->aead.data.length;
1263                 } else {
1264                         job->cipher_start_src_offset_in_bytes =
1265                                         op->sym->auth.data.offset;
1266                         job->hash_start_src_offset_in_bytes =
1267                                         op->sym->auth.data.offset;
1268                         job->msg_len_to_cipher_in_bytes = 0;
1269                         job->msg_len_to_hash_in_bytes = 0;
1270                 }
1271
1272                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1273                                 session->iv.offset);
1274                 break;
1275
1276         default:
1277                 job->cipher_start_src_offset_in_bytes =
1278                                 op->sym->cipher.data.offset;
1279                 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
1280
1281                 job->hash_start_src_offset_in_bytes = auth_start_offset(op,
1282                                 session, oop);
1283                 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
1284
1285                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1286                         session->iv.offset);
1287         }
1288
1289         /* Set user data to be crypto operation data struct */
1290         job->user_data = op;
1291
1292         return 0;
1293 }
1294
1295 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1296 /**
1297  * Process a crypto operation containing a security op and complete a
1298  * JOB_AES_HMAC job structure for submission to the multi buffer library for
1299  * processing.
1300  */
1301 static inline int
1302 set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
1303                 struct rte_crypto_op *op, uint8_t *digest_idx)
1304 {
1305         struct rte_mbuf *m_src, *m_dst;
1306         struct rte_crypto_sym_op *sym;
1307         struct aesni_mb_session *session;
1308
1309         session = get_session(qp, op);
1310         if (unlikely(session == NULL)) {
1311                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1312                 return -1;
1313         }
1314
1315         /* Only DOCSIS protocol operations supported now */
1316         if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||
1317                         session->auth.algo != IMB_AUTH_DOCSIS_CRC32) {
1318                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1319                 return -1;
1320         }
1321
1322         sym = op->sym;
1323         m_src = sym->m_src;
1324
1325         if (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {
1326                 /* in-place operation */
1327                 m_dst = m_src;
1328         } else {
1329                 /* out-of-place operation not supported */
1330                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1331                 return -ENOTSUP;
1332         }
1333
1334         /* Set crypto operation */
1335         job->chain_order = session->chain_order;
1336
1337         /* Set cipher parameters */
1338         job->cipher_direction = session->cipher.direction;
1339         job->cipher_mode = session->cipher.mode;
1340
1341         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
1342         job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
1343         job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
1344
1345         /* Set IV parameters */
1346         job->iv_len_in_bytes = session->iv.length;
1347         job->iv = (uint8_t *)op + session->iv.offset;
1348
1349         /* Set authentication parameters */
1350         job->hash_alg = session->auth.algo;
1351
1352         /* Set digest output location */
1353         job->auth_tag_output = qp->temp_digests[*digest_idx];
1354         *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1355
1356         /* Set digest length */
1357         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1358
1359         /* Set data parameters */
1360         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1361         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,
1362                                                 sym->cipher.data.offset);
1363
1364         job->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;
1365         job->msg_len_to_cipher_in_bytes = sym->cipher.data.length;
1366
1367         job->hash_start_src_offset_in_bytes = sym->auth.data.offset;
1368         job->msg_len_to_hash_in_bytes = sym->auth.data.length;
1369
1370         job->user_data = op;
1371
1372         return 0;
1373 }
1374
1375 static inline void
1376 verify_docsis_sec_crc(JOB_AES_HMAC *job, uint8_t *status)
1377 {
1378         uint16_t crc_offset;
1379         uint8_t *crc;
1380
1381         if (!job->msg_len_to_hash_in_bytes)
1382                 return;
1383
1384         crc_offset = job->hash_start_src_offset_in_bytes +
1385                         job->msg_len_to_hash_in_bytes -
1386                         job->cipher_start_src_offset_in_bytes;
1387         crc = job->dst + crc_offset;
1388
1389         /* Verify CRC (at the end of the message) */
1390         if (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0)
1391                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1392 }
1393 #endif
1394
1395 static inline void
1396 verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
1397 {
1398         /* Verify digest if required */
1399         if (memcmp(job->auth_tag_output, digest, len) != 0)
1400                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1401 }
1402
1403 static inline void
1404 generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
1405                 struct aesni_mb_session *sess)
1406 {
1407         /* No extra copy needed */
1408         if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
1409                 return;
1410
1411         /*
1412          * This can only happen for HMAC, so only digest
1413          * for authentication algos is required
1414          */
1415         memcpy(op->sym->auth.digest.data, job->auth_tag_output,
1416                         sess->auth.req_digest_len);
1417 }
1418
1419 /**
1420  * Process a completed job and return rte_mbuf which job processed
1421  *
1422  * @param qp            Queue Pair to process
1423  * @param job   JOB_AES_HMAC job to process
1424  *
1425  * @return
1426  * - Returns processed crypto operation.
1427  * - Returns NULL on invalid job
1428  */
1429 static inline struct rte_crypto_op *
1430 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
1431 {
1432         struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
1433         struct aesni_mb_session *sess = NULL;
1434
1435 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1436         uint8_t is_docsis_sec = 0;
1437
1438         if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1439                 /*
1440                  * Assuming at this point that if it's a security type op, that
1441                  * this is for DOCSIS
1442                  */
1443                 is_docsis_sec = 1;
1444                 sess = get_sec_session_private_data(op->sym->sec_session);
1445         } else
1446 #endif
1447         {
1448                 sess = get_sym_session_private_data(op->sym->session,
1449                                                 cryptodev_driver_id);
1450         }
1451
1452         if (unlikely(sess == NULL)) {
1453                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1454                 return op;
1455         }
1456
1457         if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
1458                 switch (job->status) {
1459                 case STS_COMPLETED:
1460                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1461
1462                         if (job->hash_alg == NULL_HASH)
1463                                 break;
1464
1465                         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1466                                 if (job->hash_alg == AES_CCM ||
1467                                         (job->hash_alg == AES_GMAC &&
1468                                                 sess->cipher.mode == GCM))
1469                                         verify_digest(job,
1470                                                 op->sym->aead.digest.data,
1471                                                 sess->auth.req_digest_len,
1472                                                 &op->status);
1473 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1474                                 else if (is_docsis_sec)
1475                                         verify_docsis_sec_crc(job,
1476                                                 &op->status);
1477 #endif
1478                                 else
1479                                         verify_digest(job,
1480                                                 op->sym->auth.digest.data,
1481                                                 sess->auth.req_digest_len,
1482                                                 &op->status);
1483                         } else
1484                                 generate_digest(job, op, sess);
1485                         break;
1486                 default:
1487                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1488                 }
1489         }
1490
1491         /* Free session if a session-less crypto op */
1492         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1493                 memset(sess, 0, sizeof(struct aesni_mb_session));
1494                 memset(op->sym->session, 0,
1495                         rte_cryptodev_sym_get_existing_header_session_size(
1496                                 op->sym->session));
1497                 rte_mempool_put(qp->sess_mp_priv, sess);
1498                 rte_mempool_put(qp->sess_mp, op->sym->session);
1499                 op->sym->session = NULL;
1500         }
1501
1502         return op;
1503 }
1504
1505 static inline void
1506 post_process_mb_sync_job(JOB_AES_HMAC *job)
1507 {
1508         uint32_t *st;
1509
1510         st = job->user_data;
1511         st[0] = (job->status == STS_COMPLETED) ? 0 : EBADMSG;
1512 }
1513
1514 /**
1515  * Process a completed JOB_AES_HMAC job and keep processing jobs until
1516  * get_completed_job return NULL
1517  *
1518  * @param qp            Queue Pair to process
1519  * @param job           JOB_AES_HMAC job
1520  *
1521  * @return
1522  * - Number of processed jobs
1523  */
1524 static unsigned
1525 handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
1526                 struct rte_crypto_op **ops, uint16_t nb_ops)
1527 {
1528         struct rte_crypto_op *op = NULL;
1529         unsigned processed_jobs = 0;
1530
1531         while (job != NULL) {
1532                 op = post_process_mb_job(qp, job);
1533
1534                 if (op) {
1535                         ops[processed_jobs++] = op;
1536                         qp->stats.dequeued_count++;
1537                 } else {
1538                         qp->stats.dequeue_err_count++;
1539                         break;
1540                 }
1541                 if (processed_jobs == nb_ops)
1542                         break;
1543
1544                 job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
1545         }
1546
1547         return processed_jobs;
1548 }
1549
1550 static inline uint32_t
1551 handle_completed_sync_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr)
1552 {
1553         uint32_t i;
1554
1555         for (i = 0; job != NULL; i++, job = IMB_GET_COMPLETED_JOB(mb_mgr))
1556                 post_process_mb_sync_job(job);
1557
1558         return i;
1559 }
1560
1561 static inline uint32_t
1562 flush_mb_sync_mgr(MB_MGR *mb_mgr)
1563 {
1564         JOB_AES_HMAC *job;
1565
1566         job = IMB_FLUSH_JOB(mb_mgr);
1567         return handle_completed_sync_jobs(job, mb_mgr);
1568 }
1569
1570 static inline uint16_t
1571 flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
1572                 uint16_t nb_ops)
1573 {
1574         int processed_ops = 0;
1575
1576         /* Flush the remaining jobs */
1577         JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
1578
1579         if (job)
1580                 processed_ops += handle_completed_jobs(qp, job,
1581                                 &ops[processed_ops], nb_ops - processed_ops);
1582
1583         return processed_ops;
1584 }
1585
1586 static inline JOB_AES_HMAC *
1587 set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
1588 {
1589         job->chain_order = HASH_CIPHER;
1590         job->cipher_mode = NULL_CIPHER;
1591         job->hash_alg = NULL_HASH;
1592         job->cipher_direction = DECRYPT;
1593
1594         /* Set user data to be crypto operation data struct */
1595         job->user_data = op;
1596
1597         return job;
1598 }
1599
1600 static uint16_t
1601 aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1602                 uint16_t nb_ops)
1603 {
1604         struct aesni_mb_qp *qp = queue_pair;
1605
1606         struct rte_crypto_op *op;
1607         JOB_AES_HMAC *job;
1608
1609         int retval, processed_jobs = 0;
1610
1611         if (unlikely(nb_ops == 0))
1612                 return 0;
1613
1614         uint8_t digest_idx = qp->digest_idx;
1615         do {
1616                 /* Get next free mb job struct from mb manager */
1617                 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1618                 if (unlikely(job == NULL)) {
1619                         /* if no free mb job structs we need to flush mb_mgr */
1620                         processed_jobs += flush_mb_mgr(qp,
1621                                         &ops[processed_jobs],
1622                                         nb_ops - processed_jobs);
1623
1624                         if (nb_ops == processed_jobs)
1625                                 break;
1626
1627                         job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1628                 }
1629
1630                 /*
1631                  * Get next operation to process from ingress queue.
1632                  * There is no need to return the job to the MB_MGR
1633                  * if there are no more operations to process, since the MB_MGR
1634                  * can use that pointer again in next get_next calls.
1635                  */
1636                 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1637                 if (retval < 0)
1638                         break;
1639
1640 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1641                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1642                         retval = set_sec_mb_job_params(job, qp, op,
1643                                                 &digest_idx);
1644                 else
1645 #endif
1646                         retval = set_mb_job_params(job, qp, op, &digest_idx);
1647
1648                 if (unlikely(retval != 0)) {
1649                         qp->stats.dequeue_err_count++;
1650                         set_job_null_op(job, op);
1651                 }
1652
1653                 /* Submit job to multi-buffer for processing */
1654 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1655                 job = IMB_SUBMIT_JOB(qp->mb_mgr);
1656 #else
1657                 job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
1658 #endif
1659                 /*
1660                  * If submit returns a processed job then handle it,
1661                  * before submitting subsequent jobs
1662                  */
1663                 if (job)
1664                         processed_jobs += handle_completed_jobs(qp, job,
1665                                         &ops[processed_jobs],
1666                                         nb_ops - processed_jobs);
1667
1668         } while (processed_jobs < nb_ops);
1669
1670         qp->digest_idx = digest_idx;
1671
1672         if (processed_jobs < 1)
1673                 processed_jobs += flush_mb_mgr(qp,
1674                                 &ops[processed_jobs],
1675                                 nb_ops - processed_jobs);
1676
1677         return processed_jobs;
1678 }
1679
1680 static MB_MGR *
1681 alloc_init_mb_mgr(enum aesni_mb_vector_mode vector_mode)
1682 {
1683         MB_MGR *mb_mgr = alloc_mb_mgr(0);
1684         if (mb_mgr == NULL)
1685                 return NULL;
1686
1687         switch (vector_mode) {
1688         case RTE_AESNI_MB_SSE:
1689                 init_mb_mgr_sse(mb_mgr);
1690                 break;
1691         case RTE_AESNI_MB_AVX:
1692                 init_mb_mgr_avx(mb_mgr);
1693                 break;
1694         case RTE_AESNI_MB_AVX2:
1695                 init_mb_mgr_avx2(mb_mgr);
1696                 break;
1697         case RTE_AESNI_MB_AVX512:
1698                 init_mb_mgr_avx512(mb_mgr);
1699                 break;
1700         default:
1701                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
1702                 free_mb_mgr(mb_mgr);
1703                 return NULL;
1704         }
1705
1706         return mb_mgr;
1707 }
1708
1709 static inline void
1710 aesni_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
1711 {
1712         uint32_t i;
1713
1714         for (i = 0; i != vec->num; ++i)
1715                 vec->status[i] = err;
1716 }
1717
1718 static inline int
1719 check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)
1720 {
1721         /* no multi-seg support with current AESNI-MB PMD */
1722         if (sgl->num != 1)
1723                 return ENOTSUP;
1724         else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)
1725                 return EINVAL;
1726         return 0;
1727 }
1728
1729 static inline JOB_AES_HMAC *
1730 submit_sync_job(MB_MGR *mb_mgr)
1731 {
1732 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1733         return IMB_SUBMIT_JOB(mb_mgr);
1734 #else
1735         return IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
1736 #endif
1737 }
1738
1739 static inline uint32_t
1740 generate_sync_dgst(struct rte_crypto_sym_vec *vec,
1741         const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1742 {
1743         uint32_t i, k;
1744
1745         for (i = 0, k = 0; i != vec->num; i++) {
1746                 if (vec->status[i] == 0) {
1747                         memcpy(vec->digest[i], dgst[i], len);
1748                         k++;
1749                 }
1750         }
1751
1752         return k;
1753 }
1754
1755 static inline uint32_t
1756 verify_sync_dgst(struct rte_crypto_sym_vec *vec,
1757         const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1758 {
1759         uint32_t i, k;
1760
1761         for (i = 0, k = 0; i != vec->num; i++) {
1762                 if (vec->status[i] == 0) {
1763                         if (memcmp(vec->digest[i], dgst[i], len) != 0)
1764                                 vec->status[i] = EBADMSG;
1765                         else
1766                                 k++;
1767                 }
1768         }
1769
1770         return k;
1771 }
1772
1773 uint32_t
1774 aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
1775         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
1776         struct rte_crypto_sym_vec *vec)
1777 {
1778         int32_t ret;
1779         uint32_t i, j, k, len;
1780         void *buf;
1781         JOB_AES_HMAC *job;
1782         MB_MGR *mb_mgr;
1783         struct aesni_mb_private *priv;
1784         struct aesni_mb_session *s;
1785         uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];
1786
1787         s = get_sym_session_private_data(sess, dev->driver_id);
1788         if (s == NULL) {
1789                 aesni_mb_fill_error_code(vec, EINVAL);
1790                 return 0;
1791         }
1792
1793         /* get per-thread MB MGR, create one if needed */
1794         mb_mgr = RTE_PER_LCORE(sync_mb_mgr);
1795         if (mb_mgr == NULL) {
1796
1797                 priv = dev->data->dev_private;
1798                 mb_mgr = alloc_init_mb_mgr(priv->vector_mode);
1799                 if (mb_mgr == NULL) {
1800                         aesni_mb_fill_error_code(vec, ENOMEM);
1801                         return 0;
1802                 }
1803                 RTE_PER_LCORE(sync_mb_mgr) = mb_mgr;
1804         }
1805
1806         for (i = 0, j = 0, k = 0; i != vec->num; i++) {
1807
1808
1809                 ret = check_crypto_sgl(sofs, vec->sgl + i);
1810                 if (ret != 0) {
1811                         vec->status[i] = ret;
1812                         continue;
1813                 }
1814
1815                 buf = vec->sgl[i].vec[0].base;
1816                 len = vec->sgl[i].vec[0].len;
1817
1818                 job = IMB_GET_NEXT_JOB(mb_mgr);
1819                 if (job == NULL) {
1820                         k += flush_mb_sync_mgr(mb_mgr);
1821                         job = IMB_GET_NEXT_JOB(mb_mgr);
1822                         RTE_ASSERT(job != NULL);
1823                 }
1824
1825                 /* Submit job for processing */
1826                 set_cpu_mb_job_params(job, s, sofs, buf, len,
1827                         vec->iv[i], vec->aad[i], tmp_dgst[i],
1828                         &vec->status[i]);
1829                 job = submit_sync_job(mb_mgr);
1830                 j++;
1831
1832                 /* handle completed jobs */
1833                 k += handle_completed_sync_jobs(job, mb_mgr);
1834         }
1835
1836         /* flush remaining jobs */
1837         while (k != j)
1838                 k += flush_mb_sync_mgr(mb_mgr);
1839
1840         /* finish processing for successful jobs: check/update digest */
1841         if (k != 0) {
1842                 if (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1843                         k = verify_sync_dgst(vec,
1844                                 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1845                                 s->auth.req_digest_len);
1846                 else
1847                         k = generate_sync_dgst(vec,
1848                                 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1849                                 s->auth.req_digest_len);
1850         }
1851
1852         return k;
1853 }
1854
1855 static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
1856
1857 static uint64_t
1858 vec_mode_to_flags(enum aesni_mb_vector_mode mode)
1859 {
1860         switch (mode) {
1861         case RTE_AESNI_MB_SSE:
1862                 return RTE_CRYPTODEV_FF_CPU_SSE;
1863         case RTE_AESNI_MB_AVX:
1864                 return RTE_CRYPTODEV_FF_CPU_AVX;
1865         case RTE_AESNI_MB_AVX2:
1866                 return RTE_CRYPTODEV_FF_CPU_AVX2;
1867         case RTE_AESNI_MB_AVX512:
1868                 return RTE_CRYPTODEV_FF_CPU_AVX512;
1869         default:
1870                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", mode);
1871                 return 0;
1872         }
1873 }
1874
1875 static int
1876 cryptodev_aesni_mb_create(const char *name,
1877                         struct rte_vdev_device *vdev,
1878                         struct rte_cryptodev_pmd_init_params *init_params)
1879 {
1880         struct rte_cryptodev *dev;
1881         struct aesni_mb_private *internals;
1882         enum aesni_mb_vector_mode vector_mode;
1883         MB_MGR *mb_mgr;
1884
1885         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1886         if (dev == NULL) {
1887                 AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
1888                 return -ENODEV;
1889         }
1890
1891         /* Check CPU for supported vector instruction set */
1892         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
1893                 vector_mode = RTE_AESNI_MB_AVX512;
1894         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
1895                 vector_mode = RTE_AESNI_MB_AVX2;
1896         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
1897                 vector_mode = RTE_AESNI_MB_AVX;
1898         else
1899                 vector_mode = RTE_AESNI_MB_SSE;
1900
1901         dev->driver_id = cryptodev_driver_id;
1902         dev->dev_ops = rte_aesni_mb_pmd_ops;
1903
1904         /* register rx/tx burst functions for data path */
1905         dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
1906         dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
1907
1908         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1909                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1910                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
1911                         RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
1912                         RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
1913
1914 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1915         struct rte_security_ctx *security_instance;
1916         security_instance = rte_malloc("aesni_mb_sec",
1917                                 sizeof(struct rte_security_ctx),
1918                                 RTE_CACHE_LINE_SIZE);
1919         if (security_instance == NULL) {
1920                 AESNI_MB_LOG(ERR, "rte_security_ctx memory alloc failed");
1921                 rte_cryptodev_pmd_destroy(dev);
1922                 return -ENOMEM;
1923         }
1924
1925         security_instance->device = (void *)dev;
1926         security_instance->ops = rte_aesni_mb_pmd_sec_ops;
1927         security_instance->sess_cnt = 0;
1928         dev->security_ctx = security_instance;
1929         dev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
1930 #endif
1931
1932         /* Check CPU for support for AES instruction set */
1933         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
1934                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
1935         else
1936                 AESNI_MB_LOG(WARNING, "AES instructions not supported by CPU");
1937
1938         dev->feature_flags |= vec_mode_to_flags(vector_mode);
1939
1940         mb_mgr = alloc_init_mb_mgr(vector_mode);
1941         if (mb_mgr == NULL) {
1942 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1943                 rte_free(dev->security_ctx);
1944                 dev->security_ctx = NULL;
1945 #endif
1946                 rte_cryptodev_pmd_destroy(dev);
1947                 return -ENOMEM;
1948         }
1949
1950         /* Set vector instructions mode supported */
1951         internals = dev->data->dev_private;
1952
1953         internals->vector_mode = vector_mode;
1954         internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
1955         internals->mb_mgr = mb_mgr;
1956
1957         AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
1958                         imb_get_version_str());
1959         return 0;
1960 }
1961
1962 static int
1963 cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
1964 {
1965         struct rte_cryptodev_pmd_init_params init_params = {
1966                 "",
1967                 sizeof(struct aesni_mb_private),
1968                 rte_socket_id(),
1969                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
1970         };
1971         const char *name, *args;
1972         int retval;
1973
1974         name = rte_vdev_device_name(vdev);
1975         if (name == NULL)
1976                 return -EINVAL;
1977
1978         args = rte_vdev_device_args(vdev);
1979
1980         retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
1981         if (retval) {
1982                 AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
1983                                 args);
1984                 return -EINVAL;
1985         }
1986
1987         return cryptodev_aesni_mb_create(name, vdev, &init_params);
1988 }
1989
1990 static int
1991 cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
1992 {
1993         struct rte_cryptodev *cryptodev;
1994         struct aesni_mb_private *internals;
1995         const char *name;
1996
1997         name = rte_vdev_device_name(vdev);
1998         if (name == NULL)
1999                 return -EINVAL;
2000
2001         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2002         if (cryptodev == NULL)
2003                 return -ENODEV;
2004
2005         internals = cryptodev->data->dev_private;
2006
2007         free_mb_mgr(internals->mb_mgr);
2008         if (RTE_PER_LCORE(sync_mb_mgr)) {
2009                 free_mb_mgr(RTE_PER_LCORE(sync_mb_mgr));
2010                 RTE_PER_LCORE(sync_mb_mgr) = NULL;
2011         }
2012
2013 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
2014         rte_free(cryptodev->security_ctx);
2015         cryptodev->security_ctx = NULL;
2016 #endif
2017
2018         return rte_cryptodev_pmd_destroy(cryptodev);
2019 }
2020
2021 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
2022         .probe = cryptodev_aesni_mb_probe,
2023         .remove = cryptodev_aesni_mb_remove
2024 };
2025
2026 static struct cryptodev_driver aesni_mb_crypto_drv;
2027
2028 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
2029 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
2030 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
2031         "max_nb_queue_pairs=<int> "
2032         "socket_id=<int>");
2033 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
2034                 cryptodev_aesni_mb_pmd_drv.driver,
2035                 cryptodev_driver_id);
2036 RTE_LOG_REGISTER(aesni_mb_logtype_driver, pmd.crypto.aesni_mb, NOTICE);