drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <intel-ipsec-mb.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_hexdump.h>
   9 #include <rte_cryptodev.h>
  10 #include <rte_cryptodev_pmd.h>
  11 #include <rte_bus_vdev.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cpuflags.h>
  14
  15 #include "aesni_mb_pmd_private.h"
  16
  17 int aesni_mb_logtype_driver;
  18
  19 #define AES_CCM_DIGEST_MIN_LEN 4
  20 #define AES_CCM_DIGEST_MAX_LEN 16
  21 #define HMAC_MAX_BLOCK_SIZE 128
  22 static uint8_t cryptodev_driver_id;
  23
  24 typedef void (*hash_one_block_t)(const void *data, void *digest);
  25 typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
  26
  27 /**
  28  * Calculate the authentication pre-computes
  29  *
  30  * @param one_block_hash        Function pointer to calculate digest on ipad/opad
  31  * @param ipad                  Inner pad output byte array
  32  * @param opad                  Outer pad output byte array
  33  * @param hkey                  Authentication key
  34  * @param hkey_len              Authentication key length
  35  * @param blocksize             Block size of selected hash algo
  36  */
  37 static void
  38 calculate_auth_precomputes(hash_one_block_t one_block_hash,
  39                 uint8_t *ipad, uint8_t *opad,
  40                 const uint8_t *hkey, uint16_t hkey_len,
  41                 uint16_t blocksize)
  42 {
  43         unsigned i, length;
  44
  45         uint8_t ipad_buf[blocksize] __rte_aligned(16);
  46         uint8_t opad_buf[blocksize] __rte_aligned(16);
  47
  48         /* Setup inner and outer pads */
  49         memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
  50         memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
  51
  52         /* XOR hash key with inner and outer pads */
  53         length = hkey_len > blocksize ? blocksize : hkey_len;
  54
  55         for (i = 0; i < length; i++) {
  56                 ipad_buf[i] ^= hkey[i];
  57                 opad_buf[i] ^= hkey[i];
  58         }
  59
  60         /* Compute partial hashes */
  61         (*one_block_hash)(ipad_buf, ipad);
  62         (*one_block_hash)(opad_buf, opad);
  63
  64         /* Clean up stack */
  65         memset(ipad_buf, 0, blocksize);
  66         memset(opad_buf, 0, blocksize);
  67 }
  68
  69 /** Get xform chain order */
  70 static enum aesni_mb_operation
  71 aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
  72 {
  73         if (xform == NULL)
  74                 return AESNI_MB_OP_NOT_SUPPORTED;
  75
  76         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  77                 if (xform->next == NULL)
  78                         return AESNI_MB_OP_CIPHER_ONLY;
  79                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  80                         return AESNI_MB_OP_CIPHER_HASH;
  81         }
  82
  83         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  84                 if (xform->next == NULL)
  85                         return AESNI_MB_OP_HASH_ONLY;
  86                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
  87                         return AESNI_MB_OP_HASH_CIPHER;
  88         }
  89 #if IMB_VERSION_NUM > IMB_VERSION(0, 52, 0)
  90         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  91                 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
  92                         /*
  93                          * CCM requires to hash first and cipher later
  94                          * when encrypting
  95                          */
  96                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
  97                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
  98                         else
  99                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 100                 } else {
 101                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 102                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 103                         else
 104                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 105                 }
 106         }
 107 #else
 108         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 109                 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
 110                                 xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
 111                         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 112                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
 113                         else
 114                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
 115                 }
 116         }
 117 #endif
 118
 119         return AESNI_MB_OP_NOT_SUPPORTED;
 120 }
 121
 122 /** Set session authentication parameters */
 123 static int
 124 aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
 125                 struct aesni_mb_session *sess,
 126                 const struct rte_crypto_sym_xform *xform)
 127 {
 128         hash_one_block_t hash_oneblock_fn = NULL;
 129         unsigned int key_larger_block_size = 0;
 130         uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
 131         uint32_t auth_precompute = 1;
 132
 133         if (xform == NULL) {
 134                 sess->auth.algo = NULL_HASH;
 135                 return 0;
 136         }
 137
 138         if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
 139                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
 140                 return -1;
 141         }
 142
 143         /* Set the request digest size */
 144         sess->auth.req_digest_len = xform->auth.digest_length;
 145
 146         /* Select auth generate/verify */
 147         sess->auth.operation = xform->auth.op;
 148
 149         /* Set Authentication Parameters */
 150         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
 151                 sess->auth.algo = AES_XCBC;
 152
 153                 uint16_t xcbc_mac_digest_len =
 154                         get_truncated_digest_byte_length(AES_XCBC);
 155                 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
 156                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 157                         return -EINVAL;
 158                 }
 159                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 160
 161                 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
 162                                 sess->auth.xcbc.k1_expanded,
 163                                 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
 164                 return 0;
 165         }
 166
 167         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
 168                 uint32_t dust[4*15];
 169
 170                 sess->auth.algo = AES_CMAC;
 171
 172                 uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
 173
 174                 if (sess->auth.req_digest_len > cmac_digest_len) {
 175                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 176                         return -EINVAL;
 177                 }
 178                 /*
 179                  * Multi-buffer lib supports digest sizes from 4 to 16 bytes
 180                  * in version 0.50 and sizes of 12 and 16 bytes,
 181                  * in version 0.49.
 182                  * If size requested is different, generate the full digest
 183                  * (16 bytes) in a temporary location and then memcpy
 184                  * the requested number of bytes.
 185                  */
 186                 if (sess->auth.req_digest_len < 4)
 187                         sess->auth.gen_digest_len = cmac_digest_len;
 188                 else
 189                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 190
 191                 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
 192                                 sess->auth.cmac.expkey, dust);
 193                 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
 194                                 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
 195                 return 0;
 196         }
 197
 198         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 199                 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 200                         sess->cipher.direction = ENCRYPT;
 201                         sess->chain_order = CIPHER_HASH;
 202                 } else
 203                         sess->cipher.direction = DECRYPT;
 204
 205                 sess->auth.algo = AES_GMAC;
 206                 /*
 207                  * Multi-buffer lib supports 8, 12 and 16 bytes of digest.
 208                  * If size requested is different, generate the full digest
 209                  * (16 bytes) in a temporary location and then memcpy
 210                  * the requested number of bytes.
 211                  */
 212                 if (sess->auth.req_digest_len != 16 &&
 213                                 sess->auth.req_digest_len != 12 &&
 214                                 sess->auth.req_digest_len != 8) {
 215                         sess->auth.gen_digest_len = 16;
 216                 } else {
 217                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 218                 }
 219                 sess->iv.length = xform->auth.iv.length;
 220                 sess->iv.offset = xform->auth.iv.offset;
 221
 222                 switch (xform->auth.key.length) {
 223                 case AES_128_BYTES:
 224                         IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
 225                                 &sess->cipher.gcm_key);
 226                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 227                         break;
 228                 case AES_192_BYTES:
 229                         IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
 230                                 &sess->cipher.gcm_key);
 231                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 232                         break;
 233                 case AES_256_BYTES:
 234                         IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
 235                                 &sess->cipher.gcm_key);
 236                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 237                         break;
 238                 default:
 239                         RTE_LOG(ERR, PMD, "failed to parse test type\n");
 240                         return -EINVAL;
 241                 }
 242
 243                 return 0;
 244         }
 245
 246         switch (xform->auth.algo) {
 247         case RTE_CRYPTO_AUTH_MD5_HMAC:
 248                 sess->auth.algo = MD5;
 249                 hash_oneblock_fn = mb_mgr->md5_one_block;
 250                 break;
 251         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 252                 sess->auth.algo = SHA1;
 253                 hash_oneblock_fn = mb_mgr->sha1_one_block;
 254                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
 255                         IMB_SHA1(mb_mgr,
 256                                 xform->auth.key.data,
 257                                 xform->auth.key.length,
 258                                 hashed_key);
 259                         key_larger_block_size = 1;
 260                 }
 261                 break;
 262         case RTE_CRYPTO_AUTH_SHA1:
 263                 sess->auth.algo = PLAIN_SHA1;
 264                 auth_precompute = 0;
 265                 break;
 266         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 267                 sess->auth.algo = SHA_224;
 268                 hash_oneblock_fn = mb_mgr->sha224_one_block;
 269                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
 270                         IMB_SHA224(mb_mgr,
 271                                 xform->auth.key.data,
 272                                 xform->auth.key.length,
 273                                 hashed_key);
 274                         key_larger_block_size = 1;
 275                 }
 276                 break;
 277         case RTE_CRYPTO_AUTH_SHA224:
 278                 sess->auth.algo = PLAIN_SHA_224;
 279                 auth_precompute = 0;
 280                 break;
 281         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 282                 sess->auth.algo = SHA_256;
 283                 hash_oneblock_fn = mb_mgr->sha256_one_block;
 284                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
 285                         IMB_SHA256(mb_mgr,
 286                                 xform->auth.key.data,
 287                                 xform->auth.key.length,
 288                                 hashed_key);
 289                         key_larger_block_size = 1;
 290                 }
 291                 break;
 292         case RTE_CRYPTO_AUTH_SHA256:
 293                 sess->auth.algo = PLAIN_SHA_256;
 294                 auth_precompute = 0;
 295                 break;
 296         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 297                 sess->auth.algo = SHA_384;
 298                 hash_oneblock_fn = mb_mgr->sha384_one_block;
 299                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
 300                         IMB_SHA384(mb_mgr,
 301                                 xform->auth.key.data,
 302                                 xform->auth.key.length,
 303                                 hashed_key);
 304                         key_larger_block_size = 1;
 305                 }
 306                 break;
 307         case RTE_CRYPTO_AUTH_SHA384:
 308                 sess->auth.algo = PLAIN_SHA_384;
 309                 auth_precompute = 0;
 310                 break;
 311         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 312                 sess->auth.algo = SHA_512;
 313                 hash_oneblock_fn = mb_mgr->sha512_one_block;
 314                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
 315                         IMB_SHA512(mb_mgr,
 316                                 xform->auth.key.data,
 317                                 xform->auth.key.length,
 318                                 hashed_key);
 319                         key_larger_block_size = 1;
 320                 }
 321                 break;
 322         case RTE_CRYPTO_AUTH_SHA512:
 323                 sess->auth.algo = PLAIN_SHA_512;
 324                 auth_precompute = 0;
 325                 break;
 326         default:
 327                 AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
 328                 return -ENOTSUP;
 329         }
 330         uint16_t trunc_digest_size =
 331                         get_truncated_digest_byte_length(sess->auth.algo);
 332         uint16_t full_digest_size =
 333                         get_digest_byte_length(sess->auth.algo);
 334
 335         if (sess->auth.req_digest_len > full_digest_size ||
 336                         sess->auth.req_digest_len == 0) {
 337                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 338                 return -EINVAL;
 339         }
 340
 341         if (sess->auth.req_digest_len != trunc_digest_size &&
 342                         sess->auth.req_digest_len != full_digest_size)
 343                 sess->auth.gen_digest_len = full_digest_size;
 344         else
 345                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 346
 347         /* Plain SHA does not require precompute key */
 348         if (auth_precompute == 0)
 349                 return 0;
 350
 351         /* Calculate Authentication precomputes */
 352         if (key_larger_block_size) {
 353                 calculate_auth_precomputes(hash_oneblock_fn,
 354                         sess->auth.pads.inner, sess->auth.pads.outer,
 355                         hashed_key,
 356                         xform->auth.key.length,
 357                         get_auth_algo_blocksize(sess->auth.algo));
 358         } else {
 359                 calculate_auth_precomputes(hash_oneblock_fn,
 360                         sess->auth.pads.inner, sess->auth.pads.outer,
 361                         xform->auth.key.data,
 362                         xform->auth.key.length,
 363                         get_auth_algo_blocksize(sess->auth.algo));
 364         }
 365
 366         return 0;
 367 }
 368
 369 /** Set session cipher parameters */
 370 static int
 371 aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
 372                 struct aesni_mb_session *sess,
 373                 const struct rte_crypto_sym_xform *xform)
 374 {
 375         uint8_t is_aes = 0;
 376         uint8_t is_3DES = 0;
 377
 378         if (xform == NULL) {
 379                 sess->cipher.mode = NULL_CIPHER;
 380                 return 0;
 381         }
 382
 383         if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
 384                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
 385                 return -EINVAL;
 386         }
 387
 388         /* Select cipher direction */
 389         switch (xform->cipher.op) {
 390         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
 391                 sess->cipher.direction = ENCRYPT;
 392                 break;
 393         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
 394                 sess->cipher.direction = DECRYPT;
 395                 break;
 396         default:
 397                 AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
 398                 return -EINVAL;
 399         }
 400
 401         /* Select cipher mode */
 402         switch (xform->cipher.algo) {
 403         case RTE_CRYPTO_CIPHER_AES_CBC:
 404                 sess->cipher.mode = CBC;
 405                 is_aes = 1;
 406                 break;
 407         case RTE_CRYPTO_CIPHER_AES_CTR:
 408                 sess->cipher.mode = CNTR;
 409                 is_aes = 1;
 410                 break;
 411         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 412                 sess->cipher.mode = DOCSIS_SEC_BPI;
 413                 is_aes = 1;
 414                 break;
 415         case RTE_CRYPTO_CIPHER_DES_CBC:
 416                 sess->cipher.mode = DES;
 417                 break;
 418         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 419                 sess->cipher.mode = DOCSIS_DES;
 420                 break;
 421         case RTE_CRYPTO_CIPHER_3DES_CBC:
 422                 sess->cipher.mode = DES3;
 423                 is_3DES = 1;
 424                 break;
 425         default:
 426                 AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
 427                 return -ENOTSUP;
 428         }
 429
 430         /* Set IV parameters */
 431         sess->iv.offset = xform->cipher.iv.offset;
 432         sess->iv.length = xform->cipher.iv.length;
 433
 434         /* Check key length and choose key expansion function for AES */
 435         if (is_aes) {
 436                 switch (xform->cipher.key.length) {
 437                 case AES_128_BYTES:
 438                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 439                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 440                                         sess->cipher.expanded_aes_keys.encode,
 441                                         sess->cipher.expanded_aes_keys.decode);
 442                         break;
 443                 case AES_192_BYTES:
 444                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 445                         IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
 446                                         sess->cipher.expanded_aes_keys.encode,
 447                                         sess->cipher.expanded_aes_keys.decode);
 448                         break;
 449                 case AES_256_BYTES:
 450                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 451                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 452                                         sess->cipher.expanded_aes_keys.encode,
 453                                         sess->cipher.expanded_aes_keys.decode);
 454                         break;
 455                 default:
 456                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 457                         return -EINVAL;
 458                 }
 459         } else if (is_3DES) {
 460                 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
 461                                 sess->cipher.exp_3des_keys.key[1],
 462                                 sess->cipher.exp_3des_keys.key[2]};
 463
 464                 switch (xform->cipher.key.length) {
 465                 case  24:
 466                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 467                                         xform->cipher.key.data);
 468                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 469                                         xform->cipher.key.data + 8);
 470                         IMB_DES_KEYSCHED(mb_mgr, keys[2],
 471                                         xform->cipher.key.data + 16);
 472
 473                         /* Initialize keys - 24 bytes: [K1-K2-K3] */
 474                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 475                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 476                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
 477                         break;
 478                 case 16:
 479                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 480                                         xform->cipher.key.data);
 481                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 482                                         xform->cipher.key.data + 8);
 483                         /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
 484                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 485                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 486                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 487                         break;
 488                 case 8:
 489                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 490                                         xform->cipher.key.data);
 491
 492                         /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
 493                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 494                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
 495                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 496                         break;
 497                 default:
 498                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 499                         return -EINVAL;
 500                 }
 501
 502                 sess->cipher.key_length_in_bytes = 24;
 503         } else {
 504                 if (xform->cipher.key.length != 8) {
 505                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 506                         return -EINVAL;
 507                 }
 508                 sess->cipher.key_length_in_bytes = 8;
 509
 510                 IMB_DES_KEYSCHED(mb_mgr,
 511                         (uint64_t *)sess->cipher.expanded_aes_keys.encode,
 512                                 xform->cipher.key.data);
 513                 IMB_DES_KEYSCHED(mb_mgr,
 514                         (uint64_t *)sess->cipher.expanded_aes_keys.decode,
 515                                 xform->cipher.key.data);
 516         }
 517
 518         return 0;
 519 }
 520
 521 static int
 522 aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
 523                 struct aesni_mb_session *sess,
 524                 const struct rte_crypto_sym_xform *xform)
 525 {
 526         switch (xform->aead.op) {
 527         case RTE_CRYPTO_AEAD_OP_ENCRYPT:
 528                 sess->cipher.direction = ENCRYPT;
 529                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 530                 break;
 531         case RTE_CRYPTO_AEAD_OP_DECRYPT:
 532                 sess->cipher.direction = DECRYPT;
 533                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 534                 break;
 535         default:
 536                 AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
 537                 return -EINVAL;
 538         }
 539
 540         switch (xform->aead.algo) {
 541         case RTE_CRYPTO_AEAD_AES_CCM:
 542                 sess->cipher.mode = CCM;
 543                 sess->auth.algo = AES_CCM;
 544
 545                 /* Check key length and choose key expansion function for AES */
 546                 switch (xform->aead.key.length) {
 547                 case AES_128_BYTES:
 548                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 549                         IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
 550                                         sess->cipher.expanded_aes_keys.encode,
 551                                         sess->cipher.expanded_aes_keys.decode);
 552                         break;
 553                 default:
 554                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 555                         return -EINVAL;
 556                 }
 557
 558                 break;
 559
 560         case RTE_CRYPTO_AEAD_AES_GCM:
 561                 sess->cipher.mode = GCM;
 562                 sess->auth.algo = AES_GMAC;
 563
 564                 switch (xform->aead.key.length) {
 565                 case AES_128_BYTES:
 566                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 567                         IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
 568                                 &sess->cipher.gcm_key);
 569                         break;
 570                 case AES_192_BYTES:
 571                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 572                         IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
 573                                 &sess->cipher.gcm_key);
 574                         break;
 575                 case AES_256_BYTES:
 576                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 577                         IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
 578                                 &sess->cipher.gcm_key);
 579                         break;
 580                 default:
 581                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 582                         return -EINVAL;
 583                 }
 584
 585                 break;
 586
 587         default:
 588                 AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
 589                 return -ENOTSUP;
 590         }
 591
 592         /* Set IV parameters */
 593         sess->iv.offset = xform->aead.iv.offset;
 594         sess->iv.length = xform->aead.iv.length;
 595
 596         sess->auth.req_digest_len = xform->aead.digest_length;
 597         /* CCM digests must be between 4 and 16 and an even number */
 598         if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
 599                         sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
 600                         (sess->auth.req_digest_len & 1) == 1) {
 601                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 602                 return -EINVAL;
 603         }
 604         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 605
 606         return 0;
 607 }
 608
 609 /** Parse crypto xform chain and set private session parameters */
 610 int
 611 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 612                 struct aesni_mb_session *sess,
 613                 const struct rte_crypto_sym_xform *xform)
 614 {
 615         const struct rte_crypto_sym_xform *auth_xform = NULL;
 616         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 617         const struct rte_crypto_sym_xform *aead_xform = NULL;
 618         int ret;
 619
 620         /* Select Crypto operation - hash then cipher / cipher then hash */
 621         switch (aesni_mb_get_chain_order(xform)) {
 622         case AESNI_MB_OP_HASH_CIPHER:
 623                 sess->chain_order = HASH_CIPHER;
 624                 auth_xform = xform;
 625                 cipher_xform = xform->next;
 626                 break;
 627         case AESNI_MB_OP_CIPHER_HASH:
 628                 sess->chain_order = CIPHER_HASH;
 629                 auth_xform = xform->next;
 630                 cipher_xform = xform;
 631                 break;
 632         case AESNI_MB_OP_HASH_ONLY:
 633                 sess->chain_order = HASH_CIPHER;
 634                 auth_xform = xform;
 635                 cipher_xform = NULL;
 636                 break;
 637         case AESNI_MB_OP_CIPHER_ONLY:
 638                 /*
 639                  * Multi buffer library operates only at two modes,
 640                  * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
 641                  * chain order depends on cipher operation: encryption is always
 642                  * the first operation and decryption the last one.
 643                  */
 644                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 645                         sess->chain_order = CIPHER_HASH;
 646                 else
 647                         sess->chain_order = HASH_CIPHER;
 648                 auth_xform = NULL;
 649                 cipher_xform = xform;
 650                 break;
 651         case AESNI_MB_OP_AEAD_CIPHER_HASH:
 652                 sess->chain_order = CIPHER_HASH;
 653                 sess->aead.aad_len = xform->aead.aad_length;
 654                 aead_xform = xform;
 655                 break;
 656         case AESNI_MB_OP_AEAD_HASH_CIPHER:
 657                 sess->chain_order = HASH_CIPHER;
 658                 sess->aead.aad_len = xform->aead.aad_length;
 659                 aead_xform = xform;
 660                 break;
 661         case AESNI_MB_OP_NOT_SUPPORTED:
 662         default:
 663                 AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
 664                 return -ENOTSUP;
 665         }
 666
 667         /* Default IV length = 0 */
 668         sess->iv.length = 0;
 669
 670         ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
 671         if (ret != 0) {
 672                 AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
 673                 return ret;
 674         }
 675
 676         ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
 677                         cipher_xform);
 678         if (ret != 0) {
 679                 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 680                 return ret;
 681         }
 682
 683         if (aead_xform) {
 684                 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
 685                                 aead_xform);
 686                 if (ret != 0) {
 687                         AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
 688                         return ret;
 689                 }
 690         }
 691
 692         return 0;
 693 }
 694
 695 /**
 696  * burst enqueue, place crypto operations on ingress queue for processing.
 697  *
 698  * @param __qp         Queue Pair to process
 699  * @param ops          Crypto operations for processing
 700  * @param nb_ops       Number of crypto operations for processing
 701  *
 702  * @return
 703  * - Number of crypto operations enqueued
 704  */
 705 static uint16_t
 706 aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
 707                 uint16_t nb_ops)
 708 {
 709         struct aesni_mb_qp *qp = __qp;
 710
 711         unsigned int nb_enqueued;
 712
 713         nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
 714                         (void **)ops, nb_ops, NULL);
 715
 716         qp->stats.enqueued_count += nb_enqueued;
 717
 718         return nb_enqueued;
 719 }
 720
 721 /** Get multi buffer session */
 722 static inline struct aesni_mb_session *
 723 get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
 724 {
 725         struct aesni_mb_session *sess = NULL;
 726
 727         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 728                 if (likely(op->sym->session != NULL))
 729                         sess = (struct aesni_mb_session *)
 730                                         get_sym_session_private_data(
 731                                         op->sym->session,
 732                                         cryptodev_driver_id);
 733         } else {
 734                 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
 735                 void *_sess_private_data = NULL;
 736
 737                 if (_sess == NULL)
 738                         return NULL;
 739
 740                 if (rte_mempool_get(qp->sess_mp_priv,
 741                                 (void **)&_sess_private_data))
 742                         return NULL;
 743
 744                 sess = (struct aesni_mb_session *)_sess_private_data;
 745
 746                 if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
 747                                 sess, op->sym->xform) != 0)) {
 748                         rte_mempool_put(qp->sess_mp, _sess);
 749                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 750                         sess = NULL;
 751                 }
 752                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 753                 set_sym_session_private_data(op->sym->session,
 754                                 cryptodev_driver_id, _sess_private_data);
 755         }
 756
 757         if (unlikely(sess == NULL))
 758                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 759
 760         return sess;
 761 }
 762
 763 static inline uint64_t
 764 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 765                 uint32_t oop)
 766 {
 767         struct rte_mbuf *m_src, *m_dst;
 768         uint8_t *p_src, *p_dst;
 769         uintptr_t u_src, u_dst;
 770         uint32_t cipher_end, auth_end;
 771
 772         /* Only cipher then hash needs special calculation. */
 773         if (!oop || session->chain_order != CIPHER_HASH)
 774                 return op->sym->auth.data.offset;
 775
 776         m_src = op->sym->m_src;
 777         m_dst = op->sym->m_dst;
 778
 779         p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
 780         p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
 781         u_src = (uintptr_t)p_src;
 782         u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
 783
 784         /**
 785          * Copy the content between cipher offset and auth offset for generating
 786          * correct digest.
 787          */
 788         if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
 789                 memcpy(p_dst + op->sym->auth.data.offset,
 790                                 p_src + op->sym->auth.data.offset,
 791                                 op->sym->cipher.data.offset -
 792                                 op->sym->auth.data.offset);
 793
 794         /**
 795          * Copy the content between (cipher offset + length) and (auth offset +
 796          * length) for generating correct digest
 797          */
 798         cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
 799         auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
 800         if (cipher_end < auth_end)
 801                 memcpy(p_dst + cipher_end, p_src + cipher_end,
 802                                 auth_end - cipher_end);
 803
 804         /**
 805          * Since intel-ipsec-mb only supports positive values,
 806          * we need to deduct the correct offset between src and dst.
 807          */
 808
 809         return u_src < u_dst ? (u_dst - u_src) :
 810                         (UINT64_MAX - u_src + u_dst + 1);
 811 }
 812
 813 /**
 814  * Process a crypto operation and complete a JOB_AES_HMAC job structure for
 815  * submission to the multi buffer library for processing.
 816  *
 817  * @param       qp      queue pair
 818  * @param       job     JOB_AES_HMAC structure to fill
 819  * @param       m       mbuf to process
 820  *
 821  * @return
 822  * - Completed JOB_AES_HMAC structure pointer on success
 823  * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
 824  */
 825 static inline int
 826 set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 827                 struct rte_crypto_op *op, uint8_t *digest_idx)
 828 {
 829         struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
 830         struct aesni_mb_session *session;
 831         uint32_t m_offset, oop;
 832
 833         session = get_session(qp, op);
 834         if (session == NULL) {
 835                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 836                 return -1;
 837         }
 838
 839         /* Set crypto operation */
 840         job->chain_order = session->chain_order;
 841
 842         /* Set cipher parameters */
 843         job->cipher_direction = session->cipher.direction;
 844         job->cipher_mode = session->cipher.mode;
 845
 846         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
 847
 848         /* Set authentication parameters */
 849         job->hash_alg = session->auth.algo;
 850
 851         switch (job->hash_alg) {
 852         case AES_XCBC:
 853                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
 854                 job->u.XCBC._k2 = session->auth.xcbc.k2;
 855                 job->u.XCBC._k3 = session->auth.xcbc.k3;
 856
 857                 job->aes_enc_key_expanded =
 858                                 session->cipher.expanded_aes_keys.encode;
 859                 job->aes_dec_key_expanded =
 860                                 session->cipher.expanded_aes_keys.decode;
 861                 break;
 862
 863         case AES_CCM:
 864                 job->u.CCM.aad = op->sym->aead.aad.data + 18;
 865                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
 866                 job->aes_enc_key_expanded =
 867                                 session->cipher.expanded_aes_keys.encode;
 868                 job->aes_dec_key_expanded =
 869                                 session->cipher.expanded_aes_keys.decode;
 870                 break;
 871
 872         case AES_CMAC:
 873                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
 874                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
 875                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
 876                 job->aes_enc_key_expanded =
 877                                 session->cipher.expanded_aes_keys.encode;
 878                 job->aes_dec_key_expanded =
 879                                 session->cipher.expanded_aes_keys.decode;
 880                 break;
 881
 882         case AES_GMAC:
 883                 if (session->cipher.mode == GCM) {
 884                         job->u.GCM.aad = op->sym->aead.aad.data;
 885                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
 886                 } else {
 887                         /* For GMAC */
 888                         job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
 889                                         uint8_t *, op->sym->auth.data.offset);
 890                         job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
 891                         job->cipher_mode = GCM;
 892                 }
 893                 job->aes_enc_key_expanded = &session->cipher.gcm_key;
 894                 job->aes_dec_key_expanded = &session->cipher.gcm_key;
 895                 break;
 896
 897         default:
 898                 job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
 899                 job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
 900
 901                 if (job->cipher_mode == DES3) {
 902                         job->aes_enc_key_expanded =
 903                                 session->cipher.exp_3des_keys.ks_ptr;
 904                         job->aes_dec_key_expanded =
 905                                 session->cipher.exp_3des_keys.ks_ptr;
 906                 } else {
 907                         job->aes_enc_key_expanded =
 908                                 session->cipher.expanded_aes_keys.encode;
 909                         job->aes_dec_key_expanded =
 910                                 session->cipher.expanded_aes_keys.decode;
 911                 }
 912         }
 913
 914         if (!op->sym->m_dst) {
 915                 /* in-place operation */
 916                 m_dst = m_src;
 917                 oop = 0;
 918         } else if (op->sym->m_dst == op->sym->m_src) {
 919                 /* in-place operation */
 920                 m_dst = m_src;
 921                 oop = 0;
 922         } else {
 923                 /* out-of-place operation */
 924                 m_dst = op->sym->m_dst;
 925                 oop = 1;
 926         }
 927
 928         if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
 929                         session->cipher.mode == GCM))
 930                 m_offset = op->sym->aead.data.offset;
 931         else
 932                 m_offset = op->sym->cipher.data.offset;
 933
 934         /* Set digest output location */
 935         if (job->hash_alg != NULL_HASH &&
 936                         session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
 937                 job->auth_tag_output = qp->temp_digests[*digest_idx];
 938                 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
 939         } else {
 940                 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
 941                                 session->cipher.mode == GCM))
 942                         job->auth_tag_output = op->sym->aead.digest.data;
 943                 else
 944                         job->auth_tag_output = op->sym->auth.digest.data;
 945
 946                 if (session->auth.req_digest_len != session->auth.gen_digest_len) {
 947                         job->auth_tag_output = qp->temp_digests[*digest_idx];
 948                         *digest_idx = (*digest_idx + 1) % MAX_JOBS;
 949                 }
 950         }
 951         /*
 952          * Multi-buffer library current only support returning a truncated
 953          * digest length as specified in the relevant IPsec RFCs
 954          */
 955
 956         /* Set digest length */
 957         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
 958
 959         /* Set IV parameters */
 960         job->iv_len_in_bytes = session->iv.length;
 961
 962         /* Data Parameters */
 963         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
 964         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
 965
 966         switch (job->hash_alg) {
 967         case AES_CCM:
 968                 job->cipher_start_src_offset_in_bytes =
 969                                 op->sym->aead.data.offset;
 970                 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
 971                 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
 972                 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
 973
 974                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 975                         session->iv.offset + 1);
 976                 break;
 977
 978         case AES_GMAC:
 979                 if (session->cipher.mode == GCM) {
 980                         job->cipher_start_src_offset_in_bytes =
 981                                         op->sym->aead.data.offset;
 982                         job->hash_start_src_offset_in_bytes =
 983                                         op->sym->aead.data.offset;
 984                         job->msg_len_to_cipher_in_bytes =
 985                                         op->sym->aead.data.length;
 986                         job->msg_len_to_hash_in_bytes =
 987                                         op->sym->aead.data.length;
 988                 } else {
 989                         job->cipher_start_src_offset_in_bytes =
 990                                         op->sym->auth.data.offset;
 991                         job->hash_start_src_offset_in_bytes =
 992                                         op->sym->auth.data.offset;
 993                         job->msg_len_to_cipher_in_bytes = 0;
 994                         job->msg_len_to_hash_in_bytes = 0;
 995                 }
 996
 997                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 998                                 session->iv.offset);
 999                 break;
1000
1001         default:
1002                 job->cipher_start_src_offset_in_bytes =
1003                                 op->sym->cipher.data.offset;
1004                 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
1005
1006                 job->hash_start_src_offset_in_bytes = auth_start_offset(op,
1007                                 session, oop);
1008                 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
1009
1010                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1011                         session->iv.offset);
1012         }
1013
1014         /* Set user data to be crypto operation data struct */
1015         job->user_data = op;
1016
1017         return 0;
1018 }
1019
1020 static inline void
1021 verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
1022 {
1023         /* Verify digest if required */
1024         if (memcmp(job->auth_tag_output, digest, len) != 0)
1025                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1026 }
1027
1028 static inline void
1029 generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
1030                 struct aesni_mb_session *sess)
1031 {
1032         /* No extra copy needed */
1033         if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
1034                 return;
1035
1036         /*
1037          * This can only happen for HMAC, so only digest
1038          * for authentication algos is required
1039          */
1040         memcpy(op->sym->auth.digest.data, job->auth_tag_output,
1041                         sess->auth.req_digest_len);
1042 }
1043
1044 /**
1045  * Process a completed job and return rte_mbuf which job processed
1046  *
1047  * @param qp            Queue Pair to process
1048  * @param job   JOB_AES_HMAC job to process
1049  *
1050  * @return
1051  * - Returns processed crypto operation.
1052  * - Returns NULL on invalid job
1053  */
1054 static inline struct rte_crypto_op *
1055 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
1056 {
1057         struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
1058         struct aesni_mb_session *sess = get_sym_session_private_data(
1059                                                         op->sym->session,
1060                                                         cryptodev_driver_id);
1061
1062         if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
1063                 switch (job->status) {
1064                 case STS_COMPLETED:
1065                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1066
1067                         if (job->hash_alg == NULL_HASH)
1068                                 break;
1069
1070                         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1071                                 if (job->hash_alg == AES_CCM ||
1072                                         (job->hash_alg == AES_GMAC &&
1073                                                 sess->cipher.mode == GCM))
1074                                         verify_digest(job,
1075                                                 op->sym->aead.digest.data,
1076                                                 sess->auth.req_digest_len,
1077                                                 &op->status);
1078                                 else
1079                                         verify_digest(job,
1080                                                 op->sym->auth.digest.data,
1081                                                 sess->auth.req_digest_len,
1082                                                 &op->status);
1083                         } else
1084                                 generate_digest(job, op, sess);
1085                         break;
1086                 default:
1087                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1088                 }
1089         }
1090
1091         /* Free session if a session-less crypto op */
1092         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1093                 memset(sess, 0, sizeof(struct aesni_mb_session));
1094                 memset(op->sym->session, 0,
1095                         rte_cryptodev_sym_get_existing_header_session_size(
1096                                 op->sym->session));
1097                 rte_mempool_put(qp->sess_mp_priv, sess);
1098                 rte_mempool_put(qp->sess_mp, op->sym->session);
1099                 op->sym->session = NULL;
1100         }
1101
1102         return op;
1103 }
1104
1105 /**
1106  * Process a completed JOB_AES_HMAC job and keep processing jobs until
1107  * get_completed_job return NULL
1108  *
1109  * @param qp            Queue Pair to process
1110  * @param job           JOB_AES_HMAC job
1111  *
1112  * @return
1113  * - Number of processed jobs
1114  */
1115 static unsigned
1116 handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
1117                 struct rte_crypto_op **ops, uint16_t nb_ops)
1118 {
1119         struct rte_crypto_op *op = NULL;
1120         unsigned processed_jobs = 0;
1121
1122         while (job != NULL) {
1123                 op = post_process_mb_job(qp, job);
1124
1125                 if (op) {
1126                         ops[processed_jobs++] = op;
1127                         qp->stats.dequeued_count++;
1128                 } else {
1129                         qp->stats.dequeue_err_count++;
1130                         break;
1131                 }
1132                 if (processed_jobs == nb_ops)
1133                         break;
1134
1135                 job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
1136         }
1137
1138         return processed_jobs;
1139 }
1140
1141 static inline uint16_t
1142 flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
1143                 uint16_t nb_ops)
1144 {
1145         int processed_ops = 0;
1146
1147         /* Flush the remaining jobs */
1148         JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
1149
1150         if (job)
1151                 processed_ops += handle_completed_jobs(qp, job,
1152                                 &ops[processed_ops], nb_ops - processed_ops);
1153
1154         return processed_ops;
1155 }
1156
1157 static inline JOB_AES_HMAC *
1158 set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
1159 {
1160         job->chain_order = HASH_CIPHER;
1161         job->cipher_mode = NULL_CIPHER;
1162         job->hash_alg = NULL_HASH;
1163         job->cipher_direction = DECRYPT;
1164
1165         /* Set user data to be crypto operation data struct */
1166         job->user_data = op;
1167
1168         return job;
1169 }
1170
1171 static uint16_t
1172 aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1173                 uint16_t nb_ops)
1174 {
1175         struct aesni_mb_qp *qp = queue_pair;
1176
1177         struct rte_crypto_op *op;
1178         JOB_AES_HMAC *job;
1179
1180         int retval, processed_jobs = 0;
1181
1182         if (unlikely(nb_ops == 0))
1183                 return 0;
1184
1185         uint8_t digest_idx = qp->digest_idx;
1186         do {
1187                 /* Get next free mb job struct from mb manager */
1188                 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1189                 if (unlikely(job == NULL)) {
1190                         /* if no free mb job structs we need to flush mb_mgr */
1191                         processed_jobs += flush_mb_mgr(qp,
1192                                         &ops[processed_jobs],
1193                                         nb_ops - processed_jobs);
1194
1195                         if (nb_ops == processed_jobs)
1196                                 break;
1197
1198                         job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1199                 }
1200
1201                 /*
1202                  * Get next operation to process from ingress queue.
1203                  * There is no need to return the job to the MB_MGR
1204                  * if there are no more operations to process, since the MB_MGR
1205                  * can use that pointer again in next get_next calls.
1206                  */
1207                 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1208                 if (retval < 0)
1209                         break;
1210
1211                 retval = set_mb_job_params(job, qp, op, &digest_idx);
1212                 if (unlikely(retval != 0)) {
1213                         qp->stats.dequeue_err_count++;
1214                         set_job_null_op(job, op);
1215                 }
1216
1217                 /* Submit job to multi-buffer for processing */
1218 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1219                 job = IMB_SUBMIT_JOB(qp->mb_mgr);
1220 #else
1221                 job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
1222 #endif
1223                 /*
1224                  * If submit returns a processed job then handle it,
1225                  * before submitting subsequent jobs
1226                  */
1227                 if (job)
1228                         processed_jobs += handle_completed_jobs(qp, job,
1229                                         &ops[processed_jobs],
1230                                         nb_ops - processed_jobs);
1231
1232         } while (processed_jobs < nb_ops);
1233
1234         qp->digest_idx = digest_idx;
1235
1236         if (processed_jobs < 1)
1237                 processed_jobs += flush_mb_mgr(qp,
1238                                 &ops[processed_jobs],
1239                                 nb_ops - processed_jobs);
1240
1241         return processed_jobs;
1242 }
1243
1244 static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
1245
1246 static int
1247 cryptodev_aesni_mb_create(const char *name,
1248                         struct rte_vdev_device *vdev,
1249                         struct rte_cryptodev_pmd_init_params *init_params)
1250 {
1251         struct rte_cryptodev *dev;
1252         struct aesni_mb_private *internals;
1253         enum aesni_mb_vector_mode vector_mode;
1254         MB_MGR *mb_mgr;
1255
1256         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1257         if (dev == NULL) {
1258                 AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
1259                 return -ENODEV;
1260         }
1261
1262         /* Check CPU for supported vector instruction set */
1263         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
1264                 vector_mode = RTE_AESNI_MB_AVX512;
1265         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
1266                 vector_mode = RTE_AESNI_MB_AVX2;
1267         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
1268                 vector_mode = RTE_AESNI_MB_AVX;
1269         else
1270                 vector_mode = RTE_AESNI_MB_SSE;
1271
1272         dev->driver_id = cryptodev_driver_id;
1273         dev->dev_ops = rte_aesni_mb_pmd_ops;
1274
1275         /* register rx/tx burst functions for data path */
1276         dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
1277         dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
1278
1279         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1280                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1281                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
1282
1283         /* Check CPU for support for AES instruction set */
1284         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
1285                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
1286         else
1287                 AESNI_MB_LOG(WARNING, "AES instructions not supported by CPU");
1288
1289         mb_mgr = alloc_mb_mgr(0);
1290         if (mb_mgr == NULL)
1291                 return -ENOMEM;
1292
1293         switch (vector_mode) {
1294         case RTE_AESNI_MB_SSE:
1295                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
1296                 init_mb_mgr_sse(mb_mgr);
1297                 break;
1298         case RTE_AESNI_MB_AVX:
1299                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
1300                 init_mb_mgr_avx(mb_mgr);
1301                 break;
1302         case RTE_AESNI_MB_AVX2:
1303                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
1304                 init_mb_mgr_avx2(mb_mgr);
1305                 break;
1306         case RTE_AESNI_MB_AVX512:
1307                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
1308                 init_mb_mgr_avx512(mb_mgr);
1309                 break;
1310         default:
1311                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
1312                 goto error_exit;
1313         }
1314
1315         /* Set vector instructions mode supported */
1316         internals = dev->data->dev_private;
1317
1318         internals->vector_mode = vector_mode;
1319         internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
1320         internals->mb_mgr = mb_mgr;
1321
1322         AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
1323                         imb_get_version_str());
1324
1325         return 0;
1326
1327 error_exit:
1328         if (mb_mgr)
1329                 free_mb_mgr(mb_mgr);
1330
1331         rte_cryptodev_pmd_destroy(dev);
1332
1333         return -1;
1334 }
1335
1336 static int
1337 cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
1338 {
1339         struct rte_cryptodev_pmd_init_params init_params = {
1340                 "",
1341                 sizeof(struct aesni_mb_private),
1342                 rte_socket_id(),
1343                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
1344         };
1345         const char *name, *args;
1346         int retval;
1347
1348         name = rte_vdev_device_name(vdev);
1349         if (name == NULL)
1350                 return -EINVAL;
1351
1352         args = rte_vdev_device_args(vdev);
1353
1354         retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
1355         if (retval) {
1356                 AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
1357                                 args);
1358                 return -EINVAL;
1359         }
1360
1361         return cryptodev_aesni_mb_create(name, vdev, &init_params);
1362 }
1363
1364 static int
1365 cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
1366 {
1367         struct rte_cryptodev *cryptodev;
1368         struct aesni_mb_private *internals;
1369         const char *name;
1370
1371         name = rte_vdev_device_name(vdev);
1372         if (name == NULL)
1373                 return -EINVAL;
1374
1375         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
1376         if (cryptodev == NULL)
1377                 return -ENODEV;
1378
1379         internals = cryptodev->data->dev_private;
1380
1381         free_mb_mgr(internals->mb_mgr);
1382
1383         return rte_cryptodev_pmd_destroy(cryptodev);
1384 }
1385
1386 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
1387         .probe = cryptodev_aesni_mb_probe,
1388         .remove = cryptodev_aesni_mb_remove
1389 };
1390
1391 static struct cryptodev_driver aesni_mb_crypto_drv;
1392
1393 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
1394 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
1395 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
1396         "max_nb_queue_pairs=<int> "
1397         "socket_id=<int>");
1398 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
1399                 cryptodev_aesni_mb_pmd_drv.driver,
1400                 cryptodev_driver_id);
1401
1402 RTE_INIT(aesni_mb_init_log)
1403 {
1404         aesni_mb_logtype_driver = rte_log_register("pmd.crypto.aesni_mb");
1405 }