drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <intel-ipsec-mb.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_hexdump.h>
   9 #include <rte_cryptodev.h>
  10 #include <rte_cryptodev_pmd.h>
  11 #include <rte_bus_vdev.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cpuflags.h>
  14
  15 #include "rte_aesni_mb_pmd_private.h"
  16
  17 #define AES_CCM_DIGEST_MIN_LEN 4
  18 #define AES_CCM_DIGEST_MAX_LEN 16
  19 #define HMAC_MAX_BLOCK_SIZE 128
  20 static uint8_t cryptodev_driver_id;
  21
  22 typedef void (*hash_one_block_t)(const void *data, void *digest);
  23 typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
  24
  25 /**
  26  * Calculate the authentication pre-computes
  27  *
  28  * @param one_block_hash        Function pointer to calculate digest on ipad/opad
  29  * @param ipad                  Inner pad output byte array
  30  * @param opad                  Outer pad output byte array
  31  * @param hkey                  Authentication key
  32  * @param hkey_len              Authentication key length
  33  * @param blocksize             Block size of selected hash algo
  34  */
  35 static void
  36 calculate_auth_precomputes(hash_one_block_t one_block_hash,
  37                 uint8_t *ipad, uint8_t *opad,
  38                 uint8_t *hkey, uint16_t hkey_len,
  39                 uint16_t blocksize)
  40 {
  41         unsigned i, length;
  42
  43         uint8_t ipad_buf[blocksize] __rte_aligned(16);
  44         uint8_t opad_buf[blocksize] __rte_aligned(16);
  45
  46         /* Setup inner and outer pads */
  47         memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
  48         memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
  49
  50         /* XOR hash key with inner and outer pads */
  51         length = hkey_len > blocksize ? blocksize : hkey_len;
  52
  53         for (i = 0; i < length; i++) {
  54                 ipad_buf[i] ^= hkey[i];
  55                 opad_buf[i] ^= hkey[i];
  56         }
  57
  58         /* Compute partial hashes */
  59         (*one_block_hash)(ipad_buf, ipad);
  60         (*one_block_hash)(opad_buf, opad);
  61
  62         /* Clean up stack */
  63         memset(ipad_buf, 0, blocksize);
  64         memset(opad_buf, 0, blocksize);
  65 }
  66
  67 /** Get xform chain order */
  68 static enum aesni_mb_operation
  69 aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
  70 {
  71         if (xform == NULL)
  72                 return AESNI_MB_OP_NOT_SUPPORTED;
  73
  74         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  75                 if (xform->next == NULL)
  76                         return AESNI_MB_OP_CIPHER_ONLY;
  77                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  78                         return AESNI_MB_OP_CIPHER_HASH;
  79         }
  80
  81         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  82                 if (xform->next == NULL)
  83                         return AESNI_MB_OP_HASH_ONLY;
  84                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
  85                         return AESNI_MB_OP_HASH_CIPHER;
  86         }
  87
  88         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  89                 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
  90                                 xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
  91                         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
  92                                 return AESNI_MB_OP_AEAD_CIPHER_HASH;
  93                         else
  94                                 return AESNI_MB_OP_AEAD_HASH_CIPHER;
  95                 }
  96         }
  97
  98         return AESNI_MB_OP_NOT_SUPPORTED;
  99 }
 100
 101 /** Set session authentication parameters */
 102 static int
 103 aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
 104                 struct aesni_mb_session *sess,
 105                 const struct rte_crypto_sym_xform *xform)
 106 {
 107         hash_one_block_t hash_oneblock_fn;
 108         unsigned int key_larger_block_size = 0;
 109         uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
 110
 111         if (xform == NULL) {
 112                 sess->auth.algo = NULL_HASH;
 113                 return 0;
 114         }
 115
 116         if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
 117                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
 118                 return -1;
 119         }
 120
 121         /* Set the request digest size */
 122         sess->auth.req_digest_len = xform->auth.digest_length;
 123
 124         /* Select auth generate/verify */
 125         sess->auth.operation = xform->auth.op;
 126
 127         /* Set Authentication Parameters */
 128         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
 129                 sess->auth.algo = AES_XCBC;
 130
 131                 uint16_t xcbc_mac_digest_len =
 132                         get_truncated_digest_byte_length(AES_XCBC);
 133                 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
 134                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 135                         return -EINVAL;
 136                 }
 137                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 138
 139                 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
 140                                 sess->auth.xcbc.k1_expanded,
 141                                 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
 142                 return 0;
 143         }
 144
 145         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
 146                 uint32_t dust[4*15];
 147
 148                 sess->auth.algo = AES_CMAC;
 149
 150                 uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
 151
 152                 if (sess->auth.req_digest_len > cmac_digest_len) {
 153                         AESNI_MB_LOG(ERR, "Invalid digest size\n");
 154                         return -EINVAL;
 155                 }
 156                 /*
 157                  * Multi-buffer lib supports digest sizes from 4 to 16 bytes
 158                  * in version 0.50 and sizes of 12 and 16 bytes,
 159                  * in version 0.49.
 160                  * If size requested is different, generate the full digest
 161                  * (16 bytes) in a temporary location and then memcpy
 162                  * the requested number of bytes.
 163                  */
 164                 if (sess->auth.req_digest_len < 4)
 165                         sess->auth.gen_digest_len = cmac_digest_len;
 166                 else
 167                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 168
 169                 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
 170                                 sess->auth.cmac.expkey, dust);
 171                 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
 172                                 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
 173                 return 0;
 174         }
 175
 176         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 177                 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 178                         sess->cipher.direction = ENCRYPT;
 179                         sess->chain_order = CIPHER_HASH;
 180                 } else
 181                         sess->cipher.direction = DECRYPT;
 182
 183                 sess->auth.algo = AES_GMAC;
 184                 /*
 185                  * Multi-buffer lib supports 8, 12 and 16 bytes of digest.
 186                  * If size requested is different, generate the full digest
 187                  * (16 bytes) in a temporary location and then memcpy
 188                  * the requested number of bytes.
 189                  */
 190                 if (sess->auth.req_digest_len != 16 &&
 191                                 sess->auth.req_digest_len != 12 &&
 192                                 sess->auth.req_digest_len != 8) {
 193                         sess->auth.gen_digest_len = 16;
 194                 } else {
 195                         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 196                 }
 197                 sess->iv.length = xform->auth.iv.length;
 198                 sess->iv.offset = xform->auth.iv.offset;
 199
 200                 switch (xform->auth.key.length) {
 201                 case AES_128_BYTES:
 202                         IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
 203                                 &sess->cipher.gcm_key);
 204                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 205                         break;
 206                 case AES_192_BYTES:
 207                         IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
 208                                 &sess->cipher.gcm_key);
 209                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 210                         break;
 211                 case AES_256_BYTES:
 212                         IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
 213                                 &sess->cipher.gcm_key);
 214                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 215                         break;
 216                 default:
 217                         RTE_LOG(ERR, PMD, "failed to parse test type\n");
 218                         return -EINVAL;
 219                 }
 220
 221                 return 0;
 222         }
 223
 224         switch (xform->auth.algo) {
 225         case RTE_CRYPTO_AUTH_MD5_HMAC:
 226                 sess->auth.algo = MD5;
 227                 hash_oneblock_fn = mb_mgr->md5_one_block;
 228                 break;
 229         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 230                 sess->auth.algo = SHA1;
 231                 hash_oneblock_fn = mb_mgr->sha1_one_block;
 232                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
 233                         IMB_SHA1(mb_mgr,
 234                                 xform->auth.key.data,
 235                                 xform->auth.key.length,
 236                                 hashed_key);
 237                         key_larger_block_size = 1;
 238                 }
 239                 break;
 240         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 241                 sess->auth.algo = SHA_224;
 242                 hash_oneblock_fn = mb_mgr->sha224_one_block;
 243                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
 244                         IMB_SHA224(mb_mgr,
 245                                 xform->auth.key.data,
 246                                 xform->auth.key.length,
 247                                 hashed_key);
 248                         key_larger_block_size = 1;
 249                 }
 250                 break;
 251         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 252                 sess->auth.algo = SHA_256;
 253                 hash_oneblock_fn = mb_mgr->sha256_one_block;
 254                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
 255                         IMB_SHA256(mb_mgr,
 256                                 xform->auth.key.data,
 257                                 xform->auth.key.length,
 258                                 hashed_key);
 259                         key_larger_block_size = 1;
 260                 }
 261                 break;
 262         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 263                 sess->auth.algo = SHA_384;
 264                 hash_oneblock_fn = mb_mgr->sha384_one_block;
 265                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
 266                         IMB_SHA384(mb_mgr,
 267                                 xform->auth.key.data,
 268                                 xform->auth.key.length,
 269                                 hashed_key);
 270                         key_larger_block_size = 1;
 271                 }
 272                 break;
 273         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 274                 sess->auth.algo = SHA_512;
 275                 hash_oneblock_fn = mb_mgr->sha512_one_block;
 276                 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
 277                         IMB_SHA512(mb_mgr,
 278                                 xform->auth.key.data,
 279                                 xform->auth.key.length,
 280                                 hashed_key);
 281                         key_larger_block_size = 1;
 282                 }
 283                 break;
 284         default:
 285                 AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
 286                 return -ENOTSUP;
 287         }
 288         uint16_t trunc_digest_size =
 289                         get_truncated_digest_byte_length(sess->auth.algo);
 290         uint16_t full_digest_size =
 291                         get_digest_byte_length(sess->auth.algo);
 292
 293         if (sess->auth.req_digest_len > full_digest_size ||
 294                         sess->auth.req_digest_len == 0) {
 295                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 296                 return -EINVAL;
 297         }
 298
 299         if (sess->auth.req_digest_len != trunc_digest_size &&
 300                         sess->auth.req_digest_len != full_digest_size)
 301                 sess->auth.gen_digest_len = full_digest_size;
 302         else
 303                 sess->auth.gen_digest_len = sess->auth.req_digest_len;
 304
 305         /* Calculate Authentication precomputes */
 306         if (key_larger_block_size) {
 307                 calculate_auth_precomputes(hash_oneblock_fn,
 308                         sess->auth.pads.inner, sess->auth.pads.outer,
 309                         hashed_key,
 310                         xform->auth.key.length,
 311                         get_auth_algo_blocksize(sess->auth.algo));
 312         } else {
 313                 calculate_auth_precomputes(hash_oneblock_fn,
 314                         sess->auth.pads.inner, sess->auth.pads.outer,
 315                         xform->auth.key.data,
 316                         xform->auth.key.length,
 317                         get_auth_algo_blocksize(sess->auth.algo));
 318         }
 319
 320         return 0;
 321 }
 322
 323 /** Set session cipher parameters */
 324 static int
 325 aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
 326                 struct aesni_mb_session *sess,
 327                 const struct rte_crypto_sym_xform *xform)
 328 {
 329         uint8_t is_aes = 0;
 330         uint8_t is_3DES = 0;
 331
 332         if (xform == NULL) {
 333                 sess->cipher.mode = NULL_CIPHER;
 334                 return 0;
 335         }
 336
 337         if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
 338                 AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
 339                 return -EINVAL;
 340         }
 341
 342         /* Select cipher direction */
 343         switch (xform->cipher.op) {
 344         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
 345                 sess->cipher.direction = ENCRYPT;
 346                 break;
 347         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
 348                 sess->cipher.direction = DECRYPT;
 349                 break;
 350         default:
 351                 AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
 352                 return -EINVAL;
 353         }
 354
 355         /* Select cipher mode */
 356         switch (xform->cipher.algo) {
 357         case RTE_CRYPTO_CIPHER_AES_CBC:
 358                 sess->cipher.mode = CBC;
 359                 is_aes = 1;
 360                 break;
 361         case RTE_CRYPTO_CIPHER_AES_CTR:
 362                 sess->cipher.mode = CNTR;
 363                 is_aes = 1;
 364                 break;
 365         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 366                 sess->cipher.mode = DOCSIS_SEC_BPI;
 367                 is_aes = 1;
 368                 break;
 369         case RTE_CRYPTO_CIPHER_DES_CBC:
 370                 sess->cipher.mode = DES;
 371                 break;
 372         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 373                 sess->cipher.mode = DOCSIS_DES;
 374                 break;
 375         case RTE_CRYPTO_CIPHER_3DES_CBC:
 376                 sess->cipher.mode = DES3;
 377                 is_3DES = 1;
 378                 break;
 379         default:
 380                 AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
 381                 return -ENOTSUP;
 382         }
 383
 384         /* Set IV parameters */
 385         sess->iv.offset = xform->cipher.iv.offset;
 386         sess->iv.length = xform->cipher.iv.length;
 387
 388         /* Check key length and choose key expansion function for AES */
 389         if (is_aes) {
 390                 switch (xform->cipher.key.length) {
 391                 case AES_128_BYTES:
 392                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 393                         IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
 394                                         sess->cipher.expanded_aes_keys.encode,
 395                                         sess->cipher.expanded_aes_keys.decode);
 396                         break;
 397                 case AES_192_BYTES:
 398                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 399                         IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
 400                                         sess->cipher.expanded_aes_keys.encode,
 401                                         sess->cipher.expanded_aes_keys.decode);
 402                         break;
 403                 case AES_256_BYTES:
 404                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 405                         IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
 406                                         sess->cipher.expanded_aes_keys.encode,
 407                                         sess->cipher.expanded_aes_keys.decode);
 408                         break;
 409                 default:
 410                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 411                         return -EINVAL;
 412                 }
 413         } else if (is_3DES) {
 414                 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
 415                                 sess->cipher.exp_3des_keys.key[1],
 416                                 sess->cipher.exp_3des_keys.key[2]};
 417
 418                 switch (xform->cipher.key.length) {
 419                 case  24:
 420                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 421                                         xform->cipher.key.data);
 422                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 423                                         xform->cipher.key.data + 8);
 424                         IMB_DES_KEYSCHED(mb_mgr, keys[2],
 425                                         xform->cipher.key.data + 16);
 426
 427                         /* Initialize keys - 24 bytes: [K1-K2-K3] */
 428                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 429                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 430                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
 431                         break;
 432                 case 16:
 433                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 434                                         xform->cipher.key.data);
 435                         IMB_DES_KEYSCHED(mb_mgr, keys[1],
 436                                         xform->cipher.key.data + 8);
 437                         /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
 438                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 439                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 440                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 441                         break;
 442                 case 8:
 443                         IMB_DES_KEYSCHED(mb_mgr, keys[0],
 444                                         xform->cipher.key.data);
 445
 446                         /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
 447                         sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 448                         sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
 449                         sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 450                         break;
 451                 default:
 452                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 453                         return -EINVAL;
 454                 }
 455
 456                 sess->cipher.key_length_in_bytes = 24;
 457         } else {
 458                 if (xform->cipher.key.length != 8) {
 459                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 460                         return -EINVAL;
 461                 }
 462                 sess->cipher.key_length_in_bytes = 8;
 463
 464                 IMB_DES_KEYSCHED(mb_mgr,
 465                         (uint64_t *)sess->cipher.expanded_aes_keys.encode,
 466                                 xform->cipher.key.data);
 467                 IMB_DES_KEYSCHED(mb_mgr,
 468                         (uint64_t *)sess->cipher.expanded_aes_keys.decode,
 469                                 xform->cipher.key.data);
 470         }
 471
 472         return 0;
 473 }
 474
 475 static int
 476 aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
 477                 struct aesni_mb_session *sess,
 478                 const struct rte_crypto_sym_xform *xform)
 479 {
 480         switch (xform->aead.op) {
 481         case RTE_CRYPTO_AEAD_OP_ENCRYPT:
 482                 sess->cipher.direction = ENCRYPT;
 483                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 484                 break;
 485         case RTE_CRYPTO_AEAD_OP_DECRYPT:
 486                 sess->cipher.direction = DECRYPT;
 487                 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 488                 break;
 489         default:
 490                 AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
 491                 return -EINVAL;
 492         }
 493
 494         switch (xform->aead.algo) {
 495         case RTE_CRYPTO_AEAD_AES_CCM:
 496                 sess->cipher.mode = CCM;
 497                 sess->auth.algo = AES_CCM;
 498
 499                 /* Check key length and choose key expansion function for AES */
 500                 switch (xform->aead.key.length) {
 501                 case AES_128_BYTES:
 502                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 503                         IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
 504                                         sess->cipher.expanded_aes_keys.encode,
 505                                         sess->cipher.expanded_aes_keys.decode);
 506                         break;
 507                 default:
 508                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 509                         return -EINVAL;
 510                 }
 511
 512                 break;
 513
 514         case RTE_CRYPTO_AEAD_AES_GCM:
 515                 sess->cipher.mode = GCM;
 516                 sess->auth.algo = AES_GMAC;
 517
 518                 switch (xform->aead.key.length) {
 519                 case AES_128_BYTES:
 520                         sess->cipher.key_length_in_bytes = AES_128_BYTES;
 521                         IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
 522                                 &sess->cipher.gcm_key);
 523                         break;
 524                 case AES_192_BYTES:
 525                         sess->cipher.key_length_in_bytes = AES_192_BYTES;
 526                         IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
 527                                 &sess->cipher.gcm_key);
 528                         break;
 529                 case AES_256_BYTES:
 530                         sess->cipher.key_length_in_bytes = AES_256_BYTES;
 531                         IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
 532                                 &sess->cipher.gcm_key);
 533                         break;
 534                 default:
 535                         AESNI_MB_LOG(ERR, "Invalid cipher key length");
 536                         return -EINVAL;
 537                 }
 538
 539                 break;
 540
 541         default:
 542                 AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
 543                 return -ENOTSUP;
 544         }
 545
 546         /* Set IV parameters */
 547         sess->iv.offset = xform->aead.iv.offset;
 548         sess->iv.length = xform->aead.iv.length;
 549
 550         sess->auth.req_digest_len = xform->aead.digest_length;
 551         /* CCM digests must be between 4 and 16 and an even number */
 552         if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
 553                         sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
 554                         (sess->auth.req_digest_len & 1) == 1) {
 555                 AESNI_MB_LOG(ERR, "Invalid digest size\n");
 556                 return -EINVAL;
 557         }
 558         sess->auth.gen_digest_len = sess->auth.req_digest_len;
 559
 560         return 0;
 561 }
 562
 563 /** Parse crypto xform chain and set private session parameters */
 564 int
 565 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 566                 struct aesni_mb_session *sess,
 567                 const struct rte_crypto_sym_xform *xform)
 568 {
 569         const struct rte_crypto_sym_xform *auth_xform = NULL;
 570         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 571         const struct rte_crypto_sym_xform *aead_xform = NULL;
 572         int ret;
 573
 574         /* Select Crypto operation - hash then cipher / cipher then hash */
 575         switch (aesni_mb_get_chain_order(xform)) {
 576         case AESNI_MB_OP_HASH_CIPHER:
 577                 sess->chain_order = HASH_CIPHER;
 578                 auth_xform = xform;
 579                 cipher_xform = xform->next;
 580                 break;
 581         case AESNI_MB_OP_CIPHER_HASH:
 582                 sess->chain_order = CIPHER_HASH;
 583                 auth_xform = xform->next;
 584                 cipher_xform = xform;
 585                 break;
 586         case AESNI_MB_OP_HASH_ONLY:
 587                 sess->chain_order = HASH_CIPHER;
 588                 auth_xform = xform;
 589                 cipher_xform = NULL;
 590                 break;
 591         case AESNI_MB_OP_CIPHER_ONLY:
 592                 /*
 593                  * Multi buffer library operates only at two modes,
 594                  * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
 595                  * chain order depends on cipher operation: encryption is always
 596                  * the first operation and decryption the last one.
 597                  */
 598                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 599                         sess->chain_order = CIPHER_HASH;
 600                 else
 601                         sess->chain_order = HASH_CIPHER;
 602                 auth_xform = NULL;
 603                 cipher_xform = xform;
 604                 break;
 605         case AESNI_MB_OP_AEAD_CIPHER_HASH:
 606                 sess->chain_order = CIPHER_HASH;
 607                 sess->aead.aad_len = xform->aead.aad_length;
 608                 aead_xform = xform;
 609                 break;
 610         case AESNI_MB_OP_AEAD_HASH_CIPHER:
 611                 sess->chain_order = HASH_CIPHER;
 612                 sess->aead.aad_len = xform->aead.aad_length;
 613                 aead_xform = xform;
 614                 break;
 615         case AESNI_MB_OP_NOT_SUPPORTED:
 616         default:
 617                 AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
 618                 return -ENOTSUP;
 619         }
 620
 621         /* Default IV length = 0 */
 622         sess->iv.length = 0;
 623
 624         ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
 625         if (ret != 0) {
 626                 AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
 627                 return ret;
 628         }
 629
 630         ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
 631                         cipher_xform);
 632         if (ret != 0) {
 633                 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
 634                 return ret;
 635         }
 636
 637         if (aead_xform) {
 638                 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
 639                                 aead_xform);
 640                 if (ret != 0) {
 641                         AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
 642                         return ret;
 643                 }
 644         }
 645
 646         return 0;
 647 }
 648
 649 /**
 650  * burst enqueue, place crypto operations on ingress queue for processing.
 651  *
 652  * @param __qp         Queue Pair to process
 653  * @param ops          Crypto operations for processing
 654  * @param nb_ops       Number of crypto operations for processing
 655  *
 656  * @return
 657  * - Number of crypto operations enqueued
 658  */
 659 static uint16_t
 660 aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
 661                 uint16_t nb_ops)
 662 {
 663         struct aesni_mb_qp *qp = __qp;
 664
 665         unsigned int nb_enqueued;
 666
 667         nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
 668                         (void **)ops, nb_ops, NULL);
 669
 670         qp->stats.enqueued_count += nb_enqueued;
 671
 672         return nb_enqueued;
 673 }
 674
 675 /** Get multi buffer session */
 676 static inline struct aesni_mb_session *
 677 get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
 678 {
 679         struct aesni_mb_session *sess = NULL;
 680
 681         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 682                 if (likely(op->sym->session != NULL))
 683                         sess = (struct aesni_mb_session *)
 684                                         get_sym_session_private_data(
 685                                         op->sym->session,
 686                                         cryptodev_driver_id);
 687         } else {
 688                 void *_sess = NULL;
 689                 void *_sess_private_data = NULL;
 690
 691                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 692                         return NULL;
 693
 694                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
 695                         return NULL;
 696
 697                 sess = (struct aesni_mb_session *)_sess_private_data;
 698
 699                 if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
 700                                 sess, op->sym->xform) != 0)) {
 701                         rte_mempool_put(qp->sess_mp, _sess);
 702                         rte_mempool_put(qp->sess_mp, _sess_private_data);
 703                         sess = NULL;
 704                 }
 705                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 706                 set_sym_session_private_data(op->sym->session,
 707                                 cryptodev_driver_id, _sess_private_data);
 708         }
 709
 710         if (unlikely(sess == NULL))
 711                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 712
 713         return sess;
 714 }
 715
 716 /**
 717  * Process a crypto operation and complete a JOB_AES_HMAC job structure for
 718  * submission to the multi buffer library for processing.
 719  *
 720  * @param       qp      queue pair
 721  * @param       job     JOB_AES_HMAC structure to fill
 722  * @param       m       mbuf to process
 723  *
 724  * @return
 725  * - Completed JOB_AES_HMAC structure pointer on success
 726  * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
 727  */
 728 static inline int
 729 set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 730                 struct rte_crypto_op *op, uint8_t *digest_idx)
 731 {
 732         struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
 733         struct aesni_mb_session *session;
 734         uint16_t m_offset = 0;
 735
 736         session = get_session(qp, op);
 737         if (session == NULL) {
 738                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 739                 return -1;
 740         }
 741
 742         /* Set crypto operation */
 743         job->chain_order = session->chain_order;
 744
 745         /* Set cipher parameters */
 746         job->cipher_direction = session->cipher.direction;
 747         job->cipher_mode = session->cipher.mode;
 748
 749         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
 750
 751         /* Set authentication parameters */
 752         job->hash_alg = session->auth.algo;
 753
 754         switch (job->hash_alg) {
 755         case AES_XCBC:
 756                 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
 757                 job->u.XCBC._k2 = session->auth.xcbc.k2;
 758                 job->u.XCBC._k3 = session->auth.xcbc.k3;
 759
 760                 job->aes_enc_key_expanded =
 761                                 session->cipher.expanded_aes_keys.encode;
 762                 job->aes_dec_key_expanded =
 763                                 session->cipher.expanded_aes_keys.decode;
 764                 break;
 765
 766         case AES_CCM:
 767                 job->u.CCM.aad = op->sym->aead.aad.data + 18;
 768                 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
 769                 job->aes_enc_key_expanded =
 770                                 session->cipher.expanded_aes_keys.encode;
 771                 job->aes_dec_key_expanded =
 772                                 session->cipher.expanded_aes_keys.decode;
 773                 break;
 774
 775         case AES_CMAC:
 776                 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
 777                 job->u.CMAC._skey1 = session->auth.cmac.skey1;
 778                 job->u.CMAC._skey2 = session->auth.cmac.skey2;
 779                 job->aes_enc_key_expanded =
 780                                 session->cipher.expanded_aes_keys.encode;
 781                 job->aes_dec_key_expanded =
 782                                 session->cipher.expanded_aes_keys.decode;
 783                 break;
 784
 785         case AES_GMAC:
 786                 if (session->cipher.mode == GCM) {
 787                         job->u.GCM.aad = op->sym->aead.aad.data;
 788                         job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
 789                 } else {
 790                         /* For GMAC */
 791                         job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
 792                                         uint8_t *, op->sym->auth.data.offset);
 793                         job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
 794                         job->cipher_mode = GCM;
 795                 }
 796                 job->aes_enc_key_expanded = &session->cipher.gcm_key;
 797                 job->aes_dec_key_expanded = &session->cipher.gcm_key;
 798                 break;
 799
 800         default:
 801                 job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
 802                 job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
 803
 804                 if (job->cipher_mode == DES3) {
 805                         job->aes_enc_key_expanded =
 806                                 session->cipher.exp_3des_keys.ks_ptr;
 807                         job->aes_dec_key_expanded =
 808                                 session->cipher.exp_3des_keys.ks_ptr;
 809                 } else {
 810                         job->aes_enc_key_expanded =
 811                                 session->cipher.expanded_aes_keys.encode;
 812                         job->aes_dec_key_expanded =
 813                                 session->cipher.expanded_aes_keys.decode;
 814                 }
 815         }
 816
 817         /* Mutable crypto operation parameters */
 818         if (op->sym->m_dst) {
 819                 m_src = m_dst = op->sym->m_dst;
 820
 821                 /* append space for output data to mbuf */
 822                 char *odata = rte_pktmbuf_append(m_dst,
 823                                 rte_pktmbuf_data_len(op->sym->m_src));
 824                 if (odata == NULL) {
 825                         AESNI_MB_LOG(ERR, "failed to allocate space in destination "
 826                                         "mbuf for source data");
 827                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 828                         return -1;
 829                 }
 830
 831                 memcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*),
 832                                 rte_pktmbuf_data_len(op->sym->m_src));
 833         } else {
 834                 m_dst = m_src;
 835                 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
 836                                 session->cipher.mode == GCM))
 837                         m_offset = op->sym->aead.data.offset;
 838                 else
 839                         m_offset = op->sym->cipher.data.offset;
 840         }
 841
 842         /* Set digest output location */
 843         if (job->hash_alg != NULL_HASH &&
 844                         session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
 845                 job->auth_tag_output = qp->temp_digests[*digest_idx];
 846                 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
 847         } else {
 848                 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
 849                                 session->cipher.mode == GCM))
 850                         job->auth_tag_output = op->sym->aead.digest.data;
 851                 else
 852                         job->auth_tag_output = op->sym->auth.digest.data;
 853
 854                 if (session->auth.req_digest_len != session->auth.gen_digest_len) {
 855                         job->auth_tag_output = qp->temp_digests[*digest_idx];
 856                         *digest_idx = (*digest_idx + 1) % MAX_JOBS;
 857                 }
 858         }
 859         /*
 860          * Multi-buffer library current only support returning a truncated
 861          * digest length as specified in the relevant IPsec RFCs
 862          */
 863
 864         /* Set digest length */
 865         job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
 866
 867         /* Set IV parameters */
 868         job->iv_len_in_bytes = session->iv.length;
 869
 870         /* Data  Parameter */
 871         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
 872         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
 873
 874         switch (job->hash_alg) {
 875         case AES_CCM:
 876                 job->cipher_start_src_offset_in_bytes =
 877                                 op->sym->aead.data.offset;
 878                 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
 879                 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
 880                 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
 881
 882                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 883                         session->iv.offset + 1);
 884                 break;
 885
 886         case AES_GMAC:
 887                 if (session->cipher.mode == GCM) {
 888                         job->cipher_start_src_offset_in_bytes =
 889                                         op->sym->aead.data.offset;
 890                         job->hash_start_src_offset_in_bytes =
 891                                         op->sym->aead.data.offset;
 892                         job->msg_len_to_cipher_in_bytes =
 893                                         op->sym->aead.data.length;
 894                         job->msg_len_to_hash_in_bytes =
 895                                         op->sym->aead.data.length;
 896                 } else {
 897                         job->cipher_start_src_offset_in_bytes =
 898                                         op->sym->auth.data.offset;
 899                         job->hash_start_src_offset_in_bytes =
 900                                         op->sym->auth.data.offset;
 901                         job->msg_len_to_cipher_in_bytes = 0;
 902                         job->msg_len_to_hash_in_bytes = 0;
 903                 }
 904
 905                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 906                                 session->iv.offset);
 907                 break;
 908
 909         default:
 910                 job->cipher_start_src_offset_in_bytes =
 911                                 op->sym->cipher.data.offset;
 912                 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
 913
 914                 job->hash_start_src_offset_in_bytes = op->sym->auth.data.offset;
 915                 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
 916
 917                 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 918                         session->iv.offset);
 919         }
 920
 921         /* Set user data to be crypto operation data struct */
 922         job->user_data = op;
 923
 924         return 0;
 925 }
 926
 927 static inline void
 928 verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
 929 {
 930         /* Verify digest if required */
 931         if (memcmp(job->auth_tag_output, digest, len) != 0)
 932                 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 933 }
 934
 935 static inline void
 936 generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
 937                 struct aesni_mb_session *sess)
 938 {
 939         /* No extra copy neeed */
 940         if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
 941                 return;
 942
 943         /*
 944          * This can only happen for HMAC, so only digest
 945          * for authentication algos is required
 946          */
 947         memcpy(op->sym->auth.digest.data, job->auth_tag_output,
 948                         sess->auth.req_digest_len);
 949 }
 950
 951 /**
 952  * Process a completed job and return rte_mbuf which job processed
 953  *
 954  * @param qp            Queue Pair to process
 955  * @param job   JOB_AES_HMAC job to process
 956  *
 957  * @return
 958  * - Returns processed crypto operation.
 959  * - Returns NULL on invalid job
 960  */
 961 static inline struct rte_crypto_op *
 962 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
 963 {
 964         struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
 965         struct aesni_mb_session *sess = get_sym_session_private_data(
 966                                                         op->sym->session,
 967                                                         cryptodev_driver_id);
 968
 969         if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
 970                 switch (job->status) {
 971                 case STS_COMPLETED:
 972                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 973
 974                         if (job->hash_alg == NULL_HASH)
 975                                 break;
 976
 977                         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
 978                                 if (job->hash_alg == AES_CCM ||
 979                                         (job->hash_alg == AES_GMAC &&
 980                                                 sess->cipher.mode == GCM))
 981                                         verify_digest(job,
 982                                                 op->sym->aead.digest.data,
 983                                                 sess->auth.req_digest_len,
 984                                                 &op->status);
 985                                 else
 986                                         verify_digest(job,
 987                                                 op->sym->auth.digest.data,
 988                                                 sess->auth.req_digest_len,
 989                                                 &op->status);
 990                         } else
 991                                 generate_digest(job, op, sess);
 992                         break;
 993                 default:
 994                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 995                 }
 996         }
 997
 998         /* Free session if a session-less crypto op */
 999         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1000                 memset(sess, 0, sizeof(struct aesni_mb_session));
1001                 memset(op->sym->session, 0,
1002                                 rte_cryptodev_sym_get_header_session_size());
1003                 rte_mempool_put(qp->sess_mp, sess);
1004                 rte_mempool_put(qp->sess_mp, op->sym->session);
1005                 op->sym->session = NULL;
1006         }
1007
1008         return op;
1009 }
1010
1011 /**
1012  * Process a completed JOB_AES_HMAC job and keep processing jobs until
1013  * get_completed_job return NULL
1014  *
1015  * @param qp            Queue Pair to process
1016  * @param job           JOB_AES_HMAC job
1017  *
1018  * @return
1019  * - Number of processed jobs
1020  */
1021 static unsigned
1022 handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
1023                 struct rte_crypto_op **ops, uint16_t nb_ops)
1024 {
1025         struct rte_crypto_op *op = NULL;
1026         unsigned processed_jobs = 0;
1027
1028         while (job != NULL) {
1029                 op = post_process_mb_job(qp, job);
1030
1031                 if (op) {
1032                         ops[processed_jobs++] = op;
1033                         qp->stats.dequeued_count++;
1034                 } else {
1035                         qp->stats.dequeue_err_count++;
1036                         break;
1037                 }
1038                 if (processed_jobs == nb_ops)
1039                         break;
1040
1041                 job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
1042         }
1043
1044         return processed_jobs;
1045 }
1046
1047 static inline uint16_t
1048 flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
1049                 uint16_t nb_ops)
1050 {
1051         int processed_ops = 0;
1052
1053         /* Flush the remaining jobs */
1054         JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
1055
1056         if (job)
1057                 processed_ops += handle_completed_jobs(qp, job,
1058                                 &ops[processed_ops], nb_ops - processed_ops);
1059
1060         return processed_ops;
1061 }
1062
1063 static inline JOB_AES_HMAC *
1064 set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
1065 {
1066         job->chain_order = HASH_CIPHER;
1067         job->cipher_mode = NULL_CIPHER;
1068         job->hash_alg = NULL_HASH;
1069         job->cipher_direction = DECRYPT;
1070
1071         /* Set user data to be crypto operation data struct */
1072         job->user_data = op;
1073
1074         return job;
1075 }
1076
1077 static uint16_t
1078 aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1079                 uint16_t nb_ops)
1080 {
1081         struct aesni_mb_qp *qp = queue_pair;
1082
1083         struct rte_crypto_op *op;
1084         JOB_AES_HMAC *job;
1085
1086         int retval, processed_jobs = 0;
1087
1088         if (unlikely(nb_ops == 0))
1089                 return 0;
1090
1091         uint8_t digest_idx = qp->digest_idx;
1092         do {
1093                 /* Get next free mb job struct from mb manager */
1094                 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1095                 if (unlikely(job == NULL)) {
1096                         /* if no free mb job structs we need to flush mb_mgr */
1097                         processed_jobs += flush_mb_mgr(qp,
1098                                         &ops[processed_jobs],
1099                                         nb_ops - processed_jobs);
1100
1101                         if (nb_ops == processed_jobs)
1102                                 break;
1103
1104                         job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1105                 }
1106
1107                 /*
1108                  * Get next operation to process from ingress queue.
1109                  * There is no need to return the job to the MB_MGR
1110                  * if there are no more operations to process, since the MB_MGR
1111                  * can use that pointer again in next get_next calls.
1112                  */
1113                 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1114                 if (retval < 0)
1115                         break;
1116
1117                 retval = set_mb_job_params(job, qp, op, &digest_idx);
1118                 if (unlikely(retval != 0)) {
1119                         qp->stats.dequeue_err_count++;
1120                         set_job_null_op(job, op);
1121                 }
1122
1123                 /* Submit job to multi-buffer for processing */
1124 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1125                 job = IMB_SUBMIT_JOB(qp->mb_mgr);
1126 #else
1127                 job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
1128 #endif
1129                 /*
1130                  * If submit returns a processed job then handle it,
1131                  * before submitting subsequent jobs
1132                  */
1133                 if (job)
1134                         processed_jobs += handle_completed_jobs(qp, job,
1135                                         &ops[processed_jobs],
1136                                         nb_ops - processed_jobs);
1137
1138         } while (processed_jobs < nb_ops);
1139
1140         qp->digest_idx = digest_idx;
1141
1142         if (processed_jobs < 1)
1143                 processed_jobs += flush_mb_mgr(qp,
1144                                 &ops[processed_jobs],
1145                                 nb_ops - processed_jobs);
1146
1147         return processed_jobs;
1148 }
1149
1150 static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
1151
1152 static int
1153 cryptodev_aesni_mb_create(const char *name,
1154                         struct rte_vdev_device *vdev,
1155                         struct rte_cryptodev_pmd_init_params *init_params)
1156 {
1157         struct rte_cryptodev *dev;
1158         struct aesni_mb_private *internals;
1159         enum aesni_mb_vector_mode vector_mode;
1160         MB_MGR *mb_mgr;
1161
1162         /* Check CPU for support for AES instruction set */
1163         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
1164                 AESNI_MB_LOG(ERR, "AES instructions not supported by CPU");
1165                 return -EFAULT;
1166         }
1167
1168         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1169         if (dev == NULL) {
1170                 AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
1171                 return -ENODEV;
1172         }
1173
1174         /* Check CPU for supported vector instruction set */
1175         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
1176                 vector_mode = RTE_AESNI_MB_AVX512;
1177         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
1178                 vector_mode = RTE_AESNI_MB_AVX2;
1179         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
1180                 vector_mode = RTE_AESNI_MB_AVX;
1181         else
1182                 vector_mode = RTE_AESNI_MB_SSE;
1183
1184         dev->driver_id = cryptodev_driver_id;
1185         dev->dev_ops = rte_aesni_mb_pmd_ops;
1186
1187         /* register rx/tx burst functions for data path */
1188         dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
1189         dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
1190
1191         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1192                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1193                         RTE_CRYPTODEV_FF_CPU_AESNI;
1194
1195         mb_mgr = alloc_mb_mgr(0);
1196         if (mb_mgr == NULL)
1197                 return -ENOMEM;
1198
1199         switch (vector_mode) {
1200         case RTE_AESNI_MB_SSE:
1201                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
1202                 init_mb_mgr_sse(mb_mgr);
1203                 break;
1204         case RTE_AESNI_MB_AVX:
1205                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
1206                 init_mb_mgr_avx(mb_mgr);
1207                 break;
1208         case RTE_AESNI_MB_AVX2:
1209                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
1210                 init_mb_mgr_avx2(mb_mgr);
1211                 break;
1212         case RTE_AESNI_MB_AVX512:
1213                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
1214                 init_mb_mgr_avx512(mb_mgr);
1215                 break;
1216         default:
1217                 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
1218                 goto error_exit;
1219         }
1220
1221         /* Set vector instructions mode supported */
1222         internals = dev->data->dev_private;
1223
1224         internals->vector_mode = vector_mode;
1225         internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
1226         internals->mb_mgr = mb_mgr;
1227
1228         AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
1229                         imb_get_version_str());
1230
1231         return 0;
1232
1233 error_exit:
1234         if (mb_mgr)
1235                 free_mb_mgr(mb_mgr);
1236
1237         rte_cryptodev_pmd_destroy(dev);
1238
1239         return -1;
1240 }
1241
1242 static int
1243 cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
1244 {
1245         struct rte_cryptodev_pmd_init_params init_params = {
1246                 "",
1247                 sizeof(struct aesni_mb_private),
1248                 rte_socket_id(),
1249                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
1250         };
1251         const char *name, *args;
1252         int retval;
1253
1254         name = rte_vdev_device_name(vdev);
1255         if (name == NULL)
1256                 return -EINVAL;
1257
1258         args = rte_vdev_device_args(vdev);
1259
1260         retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
1261         if (retval) {
1262                 AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
1263                                 args);
1264                 return -EINVAL;
1265         }
1266
1267         return cryptodev_aesni_mb_create(name, vdev, &init_params);
1268 }
1269
1270 static int
1271 cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
1272 {
1273         struct rte_cryptodev *cryptodev;
1274         struct aesni_mb_private *internals;
1275         const char *name;
1276
1277         name = rte_vdev_device_name(vdev);
1278         if (name == NULL)
1279                 return -EINVAL;
1280
1281         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
1282         if (cryptodev == NULL)
1283                 return -ENODEV;
1284
1285         internals = cryptodev->data->dev_private;
1286
1287         free_mb_mgr(internals->mb_mgr);
1288
1289         return rte_cryptodev_pmd_destroy(cryptodev);
1290 }
1291
1292 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
1293         .probe = cryptodev_aesni_mb_probe,
1294         .remove = cryptodev_aesni_mb_remove
1295 };
1296
1297 static struct cryptodev_driver aesni_mb_crypto_drv;
1298
1299 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
1300 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
1301 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
1302         "max_nb_queue_pairs=<int> "
1303         "socket_id=<int>");
1304 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
1305                 cryptodev_aesni_mb_pmd_drv.driver,
1306                 cryptodev_driver_id);
1307
1308 RTE_INIT(aesni_mb_init_log)
1309 {
1310         aesni_mb_logtype_driver = rte_log_register("pmd.crypto.aesni_mb");
1311 }