1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2017 Intel Corporation
5 #include <intel-ipsec-mb.h>
7 #include <rte_common.h>
8 #include <rte_hexdump.h>
9 #include <rte_cryptodev.h>
10 #include <rte_cryptodev_pmd.h>
11 #include <rte_bus_vdev.h>
12 #include <rte_malloc.h>
13 #include <rte_cpuflags.h>
14 #include <rte_per_lcore.h>
16 #include "aesni_mb_pmd_private.h"
18 #define AES_CCM_DIGEST_MIN_LEN 4
19 #define AES_CCM_DIGEST_MAX_LEN 16
20 #define HMAC_MAX_BLOCK_SIZE 128
21 static uint8_t cryptodev_driver_id;
24 * Needed to support CPU-CRYPTO API (rte_cryptodev_sym_cpu_crypto_process),
25 * as we still use JOB based API even for synchronous processing.
27 static RTE_DEFINE_PER_LCORE(MB_MGR *, sync_mb_mgr);
29 typedef void (*hash_one_block_t)(const void *data, void *digest);
30 typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
33 * Calculate the authentication pre-computes
35 * @param one_block_hash Function pointer to calculate digest on ipad/opad
36 * @param ipad Inner pad output byte array
37 * @param opad Outer pad output byte array
38 * @param hkey Authentication key
39 * @param hkey_len Authentication key length
40 * @param blocksize Block size of selected hash algo
43 calculate_auth_precomputes(hash_one_block_t one_block_hash,
44 uint8_t *ipad, uint8_t *opad,
45 const uint8_t *hkey, uint16_t hkey_len,
50 uint8_t ipad_buf[blocksize] __rte_aligned(16);
51 uint8_t opad_buf[blocksize] __rte_aligned(16);
53 /* Setup inner and outer pads */
54 memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
55 memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
57 /* XOR hash key with inner and outer pads */
58 length = hkey_len > blocksize ? blocksize : hkey_len;
60 for (i = 0; i < length; i++) {
61 ipad_buf[i] ^= hkey[i];
62 opad_buf[i] ^= hkey[i];
65 /* Compute partial hashes */
66 (*one_block_hash)(ipad_buf, ipad);
67 (*one_block_hash)(opad_buf, opad);
70 memset(ipad_buf, 0, blocksize);
71 memset(opad_buf, 0, blocksize);
74 /** Get xform chain order */
75 static enum aesni_mb_operation
76 aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
79 return AESNI_MB_OP_NOT_SUPPORTED;
81 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
82 if (xform->next == NULL)
83 return AESNI_MB_OP_CIPHER_ONLY;
84 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
85 return AESNI_MB_OP_CIPHER_HASH;
88 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
89 if (xform->next == NULL)
90 return AESNI_MB_OP_HASH_ONLY;
91 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
92 return AESNI_MB_OP_HASH_CIPHER;
94 #if IMB_VERSION_NUM > IMB_VERSION(0, 52, 0)
95 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
96 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
98 * CCM requires to hash first and cipher later
101 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
102 return AESNI_MB_OP_AEAD_HASH_CIPHER;
104 return AESNI_MB_OP_AEAD_CIPHER_HASH;
106 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
107 return AESNI_MB_OP_AEAD_CIPHER_HASH;
109 return AESNI_MB_OP_AEAD_HASH_CIPHER;
113 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
114 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
115 xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
116 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
117 return AESNI_MB_OP_AEAD_CIPHER_HASH;
119 return AESNI_MB_OP_AEAD_HASH_CIPHER;
124 return AESNI_MB_OP_NOT_SUPPORTED;
127 /** Set session authentication parameters */
129 aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
130 struct aesni_mb_session *sess,
131 const struct rte_crypto_sym_xform *xform)
133 hash_one_block_t hash_oneblock_fn = NULL;
134 unsigned int key_larger_block_size = 0;
135 uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
136 uint32_t auth_precompute = 1;
139 sess->auth.algo = NULL_HASH;
143 if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
144 AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
148 /* Set the request digest size */
149 sess->auth.req_digest_len = xform->auth.digest_length;
151 /* Select auth generate/verify */
152 sess->auth.operation = xform->auth.op;
154 /* Set Authentication Parameters */
155 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
156 sess->auth.algo = AES_XCBC;
158 uint16_t xcbc_mac_digest_len =
159 get_truncated_digest_byte_length(AES_XCBC);
160 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
161 AESNI_MB_LOG(ERR, "Invalid digest size\n");
164 sess->auth.gen_digest_len = sess->auth.req_digest_len;
166 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
167 sess->auth.xcbc.k1_expanded,
168 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
172 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
175 sess->auth.algo = AES_CMAC;
177 uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
179 if (sess->auth.req_digest_len > cmac_digest_len) {
180 AESNI_MB_LOG(ERR, "Invalid digest size\n");
184 * Multi-buffer lib supports digest sizes from 4 to 16 bytes
185 * in version 0.50 and sizes of 12 and 16 bytes,
187 * If size requested is different, generate the full digest
188 * (16 bytes) in a temporary location and then memcpy
189 * the requested number of bytes.
191 if (sess->auth.req_digest_len < 4)
192 sess->auth.gen_digest_len = cmac_digest_len;
194 sess->auth.gen_digest_len = sess->auth.req_digest_len;
196 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
197 sess->auth.cmac.expkey, dust);
198 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
199 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
203 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
204 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
205 sess->cipher.direction = ENCRYPT;
206 sess->chain_order = CIPHER_HASH;
208 sess->cipher.direction = DECRYPT;
210 sess->auth.algo = AES_GMAC;
212 * Multi-buffer lib supports 8, 12 and 16 bytes of digest.
213 * If size requested is different, generate the full digest
214 * (16 bytes) in a temporary location and then memcpy
215 * the requested number of bytes.
217 if (sess->auth.req_digest_len != 16 &&
218 sess->auth.req_digest_len != 12 &&
219 sess->auth.req_digest_len != 8) {
220 sess->auth.gen_digest_len = 16;
222 sess->auth.gen_digest_len = sess->auth.req_digest_len;
224 sess->iv.length = xform->auth.iv.length;
225 sess->iv.offset = xform->auth.iv.offset;
227 switch (xform->auth.key.length) {
229 IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
230 &sess->cipher.gcm_key);
231 sess->cipher.key_length_in_bytes = AES_128_BYTES;
234 IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
235 &sess->cipher.gcm_key);
236 sess->cipher.key_length_in_bytes = AES_192_BYTES;
239 IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
240 &sess->cipher.gcm_key);
241 sess->cipher.key_length_in_bytes = AES_256_BYTES;
244 RTE_LOG(ERR, PMD, "failed to parse test type\n");
251 switch (xform->auth.algo) {
252 case RTE_CRYPTO_AUTH_MD5_HMAC:
253 sess->auth.algo = MD5;
254 hash_oneblock_fn = mb_mgr->md5_one_block;
256 case RTE_CRYPTO_AUTH_SHA1_HMAC:
257 sess->auth.algo = SHA1;
258 hash_oneblock_fn = mb_mgr->sha1_one_block;
259 if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
261 xform->auth.key.data,
262 xform->auth.key.length,
264 key_larger_block_size = 1;
267 case RTE_CRYPTO_AUTH_SHA1:
268 sess->auth.algo = PLAIN_SHA1;
271 case RTE_CRYPTO_AUTH_SHA224_HMAC:
272 sess->auth.algo = SHA_224;
273 hash_oneblock_fn = mb_mgr->sha224_one_block;
274 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
276 xform->auth.key.data,
277 xform->auth.key.length,
279 key_larger_block_size = 1;
282 case RTE_CRYPTO_AUTH_SHA224:
283 sess->auth.algo = PLAIN_SHA_224;
286 case RTE_CRYPTO_AUTH_SHA256_HMAC:
287 sess->auth.algo = SHA_256;
288 hash_oneblock_fn = mb_mgr->sha256_one_block;
289 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
291 xform->auth.key.data,
292 xform->auth.key.length,
294 key_larger_block_size = 1;
297 case RTE_CRYPTO_AUTH_SHA256:
298 sess->auth.algo = PLAIN_SHA_256;
301 case RTE_CRYPTO_AUTH_SHA384_HMAC:
302 sess->auth.algo = SHA_384;
303 hash_oneblock_fn = mb_mgr->sha384_one_block;
304 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
306 xform->auth.key.data,
307 xform->auth.key.length,
309 key_larger_block_size = 1;
312 case RTE_CRYPTO_AUTH_SHA384:
313 sess->auth.algo = PLAIN_SHA_384;
316 case RTE_CRYPTO_AUTH_SHA512_HMAC:
317 sess->auth.algo = SHA_512;
318 hash_oneblock_fn = mb_mgr->sha512_one_block;
319 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
321 xform->auth.key.data,
322 xform->auth.key.length,
324 key_larger_block_size = 1;
327 case RTE_CRYPTO_AUTH_SHA512:
328 sess->auth.algo = PLAIN_SHA_512;
332 AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
335 uint16_t trunc_digest_size =
336 get_truncated_digest_byte_length(sess->auth.algo);
337 uint16_t full_digest_size =
338 get_digest_byte_length(sess->auth.algo);
340 if (sess->auth.req_digest_len > full_digest_size ||
341 sess->auth.req_digest_len == 0) {
342 AESNI_MB_LOG(ERR, "Invalid digest size\n");
346 if (sess->auth.req_digest_len != trunc_digest_size &&
347 sess->auth.req_digest_len != full_digest_size)
348 sess->auth.gen_digest_len = full_digest_size;
350 sess->auth.gen_digest_len = sess->auth.req_digest_len;
352 /* Plain SHA does not require precompute key */
353 if (auth_precompute == 0)
356 /* Calculate Authentication precomputes */
357 if (key_larger_block_size) {
358 calculate_auth_precomputes(hash_oneblock_fn,
359 sess->auth.pads.inner, sess->auth.pads.outer,
361 xform->auth.key.length,
362 get_auth_algo_blocksize(sess->auth.algo));
364 calculate_auth_precomputes(hash_oneblock_fn,
365 sess->auth.pads.inner, sess->auth.pads.outer,
366 xform->auth.key.data,
367 xform->auth.key.length,
368 get_auth_algo_blocksize(sess->auth.algo));
374 /** Set session cipher parameters */
376 aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
377 struct aesni_mb_session *sess,
378 const struct rte_crypto_sym_xform *xform)
382 uint8_t is_docsis = 0;
385 sess->cipher.mode = NULL_CIPHER;
389 if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
390 AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
394 /* Select cipher direction */
395 switch (xform->cipher.op) {
396 case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
397 sess->cipher.direction = ENCRYPT;
399 case RTE_CRYPTO_CIPHER_OP_DECRYPT:
400 sess->cipher.direction = DECRYPT;
403 AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
407 /* Select cipher mode */
408 switch (xform->cipher.algo) {
409 case RTE_CRYPTO_CIPHER_AES_CBC:
410 sess->cipher.mode = CBC;
413 case RTE_CRYPTO_CIPHER_AES_CTR:
414 sess->cipher.mode = CNTR;
417 case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
418 sess->cipher.mode = DOCSIS_SEC_BPI;
421 case RTE_CRYPTO_CIPHER_DES_CBC:
422 sess->cipher.mode = DES;
424 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
425 sess->cipher.mode = DOCSIS_DES;
427 case RTE_CRYPTO_CIPHER_3DES_CBC:
428 sess->cipher.mode = DES3;
432 AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
436 /* Set IV parameters */
437 sess->iv.offset = xform->cipher.iv.offset;
438 sess->iv.length = xform->cipher.iv.length;
440 /* Check key length and choose key expansion function for AES */
442 switch (xform->cipher.key.length) {
444 sess->cipher.key_length_in_bytes = AES_128_BYTES;
445 IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
446 sess->cipher.expanded_aes_keys.encode,
447 sess->cipher.expanded_aes_keys.decode);
450 sess->cipher.key_length_in_bytes = AES_192_BYTES;
451 IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
452 sess->cipher.expanded_aes_keys.encode,
453 sess->cipher.expanded_aes_keys.decode);
456 sess->cipher.key_length_in_bytes = AES_256_BYTES;
457 IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
458 sess->cipher.expanded_aes_keys.encode,
459 sess->cipher.expanded_aes_keys.decode);
462 AESNI_MB_LOG(ERR, "Invalid cipher key length");
465 } else if (is_docsis) {
466 switch (xform->cipher.key.length) {
468 sess->cipher.key_length_in_bytes = AES_128_BYTES;
469 IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
470 sess->cipher.expanded_aes_keys.encode,
471 sess->cipher.expanded_aes_keys.decode);
473 #if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
475 sess->cipher.key_length_in_bytes = AES_256_BYTES;
476 IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
477 sess->cipher.expanded_aes_keys.encode,
478 sess->cipher.expanded_aes_keys.decode);
482 AESNI_MB_LOG(ERR, "Invalid cipher key length");
485 } else if (is_3DES) {
486 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
487 sess->cipher.exp_3des_keys.key[1],
488 sess->cipher.exp_3des_keys.key[2]};
490 switch (xform->cipher.key.length) {
492 IMB_DES_KEYSCHED(mb_mgr, keys[0],
493 xform->cipher.key.data);
494 IMB_DES_KEYSCHED(mb_mgr, keys[1],
495 xform->cipher.key.data + 8);
496 IMB_DES_KEYSCHED(mb_mgr, keys[2],
497 xform->cipher.key.data + 16);
499 /* Initialize keys - 24 bytes: [K1-K2-K3] */
500 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
501 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
502 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
505 IMB_DES_KEYSCHED(mb_mgr, keys[0],
506 xform->cipher.key.data);
507 IMB_DES_KEYSCHED(mb_mgr, keys[1],
508 xform->cipher.key.data + 8);
509 /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
510 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
511 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
512 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
515 IMB_DES_KEYSCHED(mb_mgr, keys[0],
516 xform->cipher.key.data);
518 /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
519 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
520 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
521 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
524 AESNI_MB_LOG(ERR, "Invalid cipher key length");
528 sess->cipher.key_length_in_bytes = 24;
530 if (xform->cipher.key.length != 8) {
531 AESNI_MB_LOG(ERR, "Invalid cipher key length");
534 sess->cipher.key_length_in_bytes = 8;
536 IMB_DES_KEYSCHED(mb_mgr,
537 (uint64_t *)sess->cipher.expanded_aes_keys.encode,
538 xform->cipher.key.data);
539 IMB_DES_KEYSCHED(mb_mgr,
540 (uint64_t *)sess->cipher.expanded_aes_keys.decode,
541 xform->cipher.key.data);
548 aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
549 struct aesni_mb_session *sess,
550 const struct rte_crypto_sym_xform *xform)
552 switch (xform->aead.op) {
553 case RTE_CRYPTO_AEAD_OP_ENCRYPT:
554 sess->cipher.direction = ENCRYPT;
555 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
557 case RTE_CRYPTO_AEAD_OP_DECRYPT:
558 sess->cipher.direction = DECRYPT;
559 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
562 AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
566 switch (xform->aead.algo) {
567 case RTE_CRYPTO_AEAD_AES_CCM:
568 sess->cipher.mode = CCM;
569 sess->auth.algo = AES_CCM;
571 /* Check key length and choose key expansion function for AES */
572 switch (xform->aead.key.length) {
574 sess->cipher.key_length_in_bytes = AES_128_BYTES;
575 IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
576 sess->cipher.expanded_aes_keys.encode,
577 sess->cipher.expanded_aes_keys.decode);
580 AESNI_MB_LOG(ERR, "Invalid cipher key length");
586 case RTE_CRYPTO_AEAD_AES_GCM:
587 sess->cipher.mode = GCM;
588 sess->auth.algo = AES_GMAC;
590 switch (xform->aead.key.length) {
592 sess->cipher.key_length_in_bytes = AES_128_BYTES;
593 IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
594 &sess->cipher.gcm_key);
597 sess->cipher.key_length_in_bytes = AES_192_BYTES;
598 IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
599 &sess->cipher.gcm_key);
602 sess->cipher.key_length_in_bytes = AES_256_BYTES;
603 IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
604 &sess->cipher.gcm_key);
607 AESNI_MB_LOG(ERR, "Invalid cipher key length");
614 AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
618 /* Set IV parameters */
619 sess->iv.offset = xform->aead.iv.offset;
620 sess->iv.length = xform->aead.iv.length;
622 sess->auth.req_digest_len = xform->aead.digest_length;
623 /* CCM digests must be between 4 and 16 and an even number */
624 if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
625 sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
626 (sess->auth.req_digest_len & 1) == 1) {
627 AESNI_MB_LOG(ERR, "Invalid digest size\n");
630 sess->auth.gen_digest_len = sess->auth.req_digest_len;
635 /** Parse crypto xform chain and set private session parameters */
637 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
638 struct aesni_mb_session *sess,
639 const struct rte_crypto_sym_xform *xform)
641 const struct rte_crypto_sym_xform *auth_xform = NULL;
642 const struct rte_crypto_sym_xform *cipher_xform = NULL;
643 const struct rte_crypto_sym_xform *aead_xform = NULL;
646 /* Select Crypto operation - hash then cipher / cipher then hash */
647 switch (aesni_mb_get_chain_order(xform)) {
648 case AESNI_MB_OP_HASH_CIPHER:
649 sess->chain_order = HASH_CIPHER;
651 cipher_xform = xform->next;
653 case AESNI_MB_OP_CIPHER_HASH:
654 sess->chain_order = CIPHER_HASH;
655 auth_xform = xform->next;
656 cipher_xform = xform;
658 case AESNI_MB_OP_HASH_ONLY:
659 sess->chain_order = HASH_CIPHER;
663 case AESNI_MB_OP_CIPHER_ONLY:
665 * Multi buffer library operates only at two modes,
666 * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
667 * chain order depends on cipher operation: encryption is always
668 * the first operation and decryption the last one.
670 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
671 sess->chain_order = CIPHER_HASH;
673 sess->chain_order = HASH_CIPHER;
675 cipher_xform = xform;
677 case AESNI_MB_OP_AEAD_CIPHER_HASH:
678 sess->chain_order = CIPHER_HASH;
679 sess->aead.aad_len = xform->aead.aad_length;
682 case AESNI_MB_OP_AEAD_HASH_CIPHER:
683 sess->chain_order = HASH_CIPHER;
684 sess->aead.aad_len = xform->aead.aad_length;
687 case AESNI_MB_OP_NOT_SUPPORTED:
689 AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
693 /* Default IV length = 0 */
696 ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
698 AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
702 ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
705 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
710 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
713 AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
722 * burst enqueue, place crypto operations on ingress queue for processing.
724 * @param __qp Queue Pair to process
725 * @param ops Crypto operations for processing
726 * @param nb_ops Number of crypto operations for processing
729 * - Number of crypto operations enqueued
732 aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
735 struct aesni_mb_qp *qp = __qp;
737 unsigned int nb_enqueued;
739 nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
740 (void **)ops, nb_ops, NULL);
742 qp->stats.enqueued_count += nb_enqueued;
747 /** Get multi buffer session */
748 static inline struct aesni_mb_session *
749 get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
751 struct aesni_mb_session *sess = NULL;
753 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
754 if (likely(op->sym->session != NULL))
755 sess = (struct aesni_mb_session *)
756 get_sym_session_private_data(
758 cryptodev_driver_id);
760 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
761 void *_sess_private_data = NULL;
766 if (rte_mempool_get(qp->sess_mp_priv,
767 (void **)&_sess_private_data))
770 sess = (struct aesni_mb_session *)_sess_private_data;
772 if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
773 sess, op->sym->xform) != 0)) {
774 rte_mempool_put(qp->sess_mp, _sess);
775 rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
778 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
779 set_sym_session_private_data(op->sym->session,
780 cryptodev_driver_id, _sess_private_data);
783 if (unlikely(sess == NULL))
784 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
789 static inline uint64_t
790 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
793 struct rte_mbuf *m_src, *m_dst;
794 uint8_t *p_src, *p_dst;
795 uintptr_t u_src, u_dst;
796 uint32_t cipher_end, auth_end;
798 /* Only cipher then hash needs special calculation. */
799 if (!oop || session->chain_order != CIPHER_HASH)
800 return op->sym->auth.data.offset;
802 m_src = op->sym->m_src;
803 m_dst = op->sym->m_dst;
805 p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
806 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
807 u_src = (uintptr_t)p_src;
808 u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
811 * Copy the content between cipher offset and auth offset for generating
814 if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
815 memcpy(p_dst + op->sym->auth.data.offset,
816 p_src + op->sym->auth.data.offset,
817 op->sym->cipher.data.offset -
818 op->sym->auth.data.offset);
821 * Copy the content between (cipher offset + length) and (auth offset +
822 * length) for generating correct digest
824 cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
825 auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
826 if (cipher_end < auth_end)
827 memcpy(p_dst + cipher_end, p_src + cipher_end,
828 auth_end - cipher_end);
831 * Since intel-ipsec-mb only supports positive values,
832 * we need to deduct the correct offset between src and dst.
835 return u_src < u_dst ? (u_dst - u_src) :
836 (UINT64_MAX - u_src + u_dst + 1);
840 set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
841 union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
842 void *iv, void *aad, void *digest, void *udata)
844 /* Set crypto operation */
845 job->chain_order = session->chain_order;
847 /* Set cipher parameters */
848 job->cipher_direction = session->cipher.direction;
849 job->cipher_mode = session->cipher.mode;
851 job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
853 /* Set authentication parameters */
854 job->hash_alg = session->auth.algo;
857 switch (job->hash_alg) {
859 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
860 job->u.XCBC._k2 = session->auth.xcbc.k2;
861 job->u.XCBC._k3 = session->auth.xcbc.k3;
863 job->aes_enc_key_expanded =
864 session->cipher.expanded_aes_keys.encode;
865 job->aes_dec_key_expanded =
866 session->cipher.expanded_aes_keys.decode;
870 job->u.CCM.aad = (uint8_t *)aad + 18;
871 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
872 job->aes_enc_key_expanded =
873 session->cipher.expanded_aes_keys.encode;
874 job->aes_dec_key_expanded =
875 session->cipher.expanded_aes_keys.decode;
880 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
881 job->u.CMAC._skey1 = session->auth.cmac.skey1;
882 job->u.CMAC._skey2 = session->auth.cmac.skey2;
883 job->aes_enc_key_expanded =
884 session->cipher.expanded_aes_keys.encode;
885 job->aes_dec_key_expanded =
886 session->cipher.expanded_aes_keys.decode;
890 if (session->cipher.mode == GCM) {
891 job->u.GCM.aad = aad;
892 job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
895 job->u.GCM.aad = buf;
896 job->u.GCM.aad_len_in_bytes = len;
897 job->cipher_mode = GCM;
899 job->aes_enc_key_expanded = &session->cipher.gcm_key;
900 job->aes_dec_key_expanded = &session->cipher.gcm_key;
904 job->u.HMAC._hashed_auth_key_xor_ipad =
905 session->auth.pads.inner;
906 job->u.HMAC._hashed_auth_key_xor_opad =
907 session->auth.pads.outer;
909 if (job->cipher_mode == DES3) {
910 job->aes_enc_key_expanded =
911 session->cipher.exp_3des_keys.ks_ptr;
912 job->aes_dec_key_expanded =
913 session->cipher.exp_3des_keys.ks_ptr;
915 job->aes_enc_key_expanded =
916 session->cipher.expanded_aes_keys.encode;
917 job->aes_dec_key_expanded =
918 session->cipher.expanded_aes_keys.decode;
923 * Multi-buffer library current only support returning a truncated
924 * digest length as specified in the relevant IPsec RFCs
927 /* Set digest location and length */
928 job->auth_tag_output = digest;
929 job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
931 /* Set IV parameters */
932 job->iv_len_in_bytes = session->iv.length;
934 /* Data Parameters */
936 job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;
937 job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;
938 job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;
939 if (job->hash_alg == AES_GMAC && session->cipher.mode != GCM) {
940 job->msg_len_to_hash_in_bytes = 0;
941 job->msg_len_to_cipher_in_bytes = 0;
943 job->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -
945 job->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -
946 sofs.ofs.cipher.tail;
949 job->user_data = udata;
953 * Process a crypto operation and complete a JOB_AES_HMAC job structure for
954 * submission to the multi buffer library for processing.
956 * @param qp queue pair
957 * @param job JOB_AES_HMAC structure to fill
958 * @param m mbuf to process
961 * - Completed JOB_AES_HMAC structure pointer on success
962 * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
965 set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
966 struct rte_crypto_op *op, uint8_t *digest_idx)
968 struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
969 struct aesni_mb_session *session;
970 uint32_t m_offset, oop;
972 session = get_session(qp, op);
973 if (session == NULL) {
974 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
978 /* Set crypto operation */
979 job->chain_order = session->chain_order;
981 /* Set cipher parameters */
982 job->cipher_direction = session->cipher.direction;
983 job->cipher_mode = session->cipher.mode;
985 job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
987 /* Set authentication parameters */
988 job->hash_alg = session->auth.algo;
990 switch (job->hash_alg) {
992 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
993 job->u.XCBC._k2 = session->auth.xcbc.k2;
994 job->u.XCBC._k3 = session->auth.xcbc.k3;
996 job->aes_enc_key_expanded =
997 session->cipher.expanded_aes_keys.encode;
998 job->aes_dec_key_expanded =
999 session->cipher.expanded_aes_keys.decode;
1003 job->u.CCM.aad = op->sym->aead.aad.data + 18;
1004 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1005 job->aes_enc_key_expanded =
1006 session->cipher.expanded_aes_keys.encode;
1007 job->aes_dec_key_expanded =
1008 session->cipher.expanded_aes_keys.decode;
1012 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1013 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1014 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1015 job->aes_enc_key_expanded =
1016 session->cipher.expanded_aes_keys.encode;
1017 job->aes_dec_key_expanded =
1018 session->cipher.expanded_aes_keys.decode;
1022 if (session->cipher.mode == GCM) {
1023 job->u.GCM.aad = op->sym->aead.aad.data;
1024 job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1027 job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
1028 uint8_t *, op->sym->auth.data.offset);
1029 job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
1030 job->cipher_mode = GCM;
1032 job->aes_enc_key_expanded = &session->cipher.gcm_key;
1033 job->aes_dec_key_expanded = &session->cipher.gcm_key;
1037 job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
1038 job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
1040 if (job->cipher_mode == DES3) {
1041 job->aes_enc_key_expanded =
1042 session->cipher.exp_3des_keys.ks_ptr;
1043 job->aes_dec_key_expanded =
1044 session->cipher.exp_3des_keys.ks_ptr;
1046 job->aes_enc_key_expanded =
1047 session->cipher.expanded_aes_keys.encode;
1048 job->aes_dec_key_expanded =
1049 session->cipher.expanded_aes_keys.decode;
1053 if (!op->sym->m_dst) {
1054 /* in-place operation */
1057 } else if (op->sym->m_dst == op->sym->m_src) {
1058 /* in-place operation */
1062 /* out-of-place operation */
1063 m_dst = op->sym->m_dst;
1067 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
1068 session->cipher.mode == GCM))
1069 m_offset = op->sym->aead.data.offset;
1071 m_offset = op->sym->cipher.data.offset;
1073 /* Set digest output location */
1074 if (job->hash_alg != NULL_HASH &&
1075 session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1076 job->auth_tag_output = qp->temp_digests[*digest_idx];
1077 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1079 if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
1080 session->cipher.mode == GCM))
1081 job->auth_tag_output = op->sym->aead.digest.data;
1083 job->auth_tag_output = op->sym->auth.digest.data;
1085 if (session->auth.req_digest_len != session->auth.gen_digest_len) {
1086 job->auth_tag_output = qp->temp_digests[*digest_idx];
1087 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
1091 * Multi-buffer library current only support returning a truncated
1092 * digest length as specified in the relevant IPsec RFCs
1095 /* Set digest length */
1096 job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1098 /* Set IV parameters */
1099 job->iv_len_in_bytes = session->iv.length;
1101 /* Data Parameters */
1102 job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1103 job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
1105 switch (job->hash_alg) {
1107 job->cipher_start_src_offset_in_bytes =
1108 op->sym->aead.data.offset;
1109 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
1110 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
1111 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
1113 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1114 session->iv.offset + 1);
1118 if (session->cipher.mode == GCM) {
1119 job->cipher_start_src_offset_in_bytes =
1120 op->sym->aead.data.offset;
1121 job->hash_start_src_offset_in_bytes =
1122 op->sym->aead.data.offset;
1123 job->msg_len_to_cipher_in_bytes =
1124 op->sym->aead.data.length;
1125 job->msg_len_to_hash_in_bytes =
1126 op->sym->aead.data.length;
1128 job->cipher_start_src_offset_in_bytes =
1129 op->sym->auth.data.offset;
1130 job->hash_start_src_offset_in_bytes =
1131 op->sym->auth.data.offset;
1132 job->msg_len_to_cipher_in_bytes = 0;
1133 job->msg_len_to_hash_in_bytes = 0;
1136 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1137 session->iv.offset);
1141 job->cipher_start_src_offset_in_bytes =
1142 op->sym->cipher.data.offset;
1143 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
1145 job->hash_start_src_offset_in_bytes = auth_start_offset(op,
1147 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
1149 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1150 session->iv.offset);
1153 /* Set user data to be crypto operation data struct */
1154 job->user_data = op;
1160 verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
1162 /* Verify digest if required */
1163 if (memcmp(job->auth_tag_output, digest, len) != 0)
1164 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1168 generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
1169 struct aesni_mb_session *sess)
1171 /* No extra copy needed */
1172 if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
1176 * This can only happen for HMAC, so only digest
1177 * for authentication algos is required
1179 memcpy(op->sym->auth.digest.data, job->auth_tag_output,
1180 sess->auth.req_digest_len);
1184 * Process a completed job and return rte_mbuf which job processed
1186 * @param qp Queue Pair to process
1187 * @param job JOB_AES_HMAC job to process
1190 * - Returns processed crypto operation.
1191 * - Returns NULL on invalid job
1193 static inline struct rte_crypto_op *
1194 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
1196 struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
1197 struct aesni_mb_session *sess = get_sym_session_private_data(
1199 cryptodev_driver_id);
1200 if (unlikely(sess == NULL)) {
1201 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1205 if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
1206 switch (job->status) {
1208 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1210 if (job->hash_alg == NULL_HASH)
1213 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1214 if (job->hash_alg == AES_CCM ||
1215 (job->hash_alg == AES_GMAC &&
1216 sess->cipher.mode == GCM))
1218 op->sym->aead.digest.data,
1219 sess->auth.req_digest_len,
1223 op->sym->auth.digest.data,
1224 sess->auth.req_digest_len,
1227 generate_digest(job, op, sess);
1230 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1234 /* Free session if a session-less crypto op */
1235 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1236 memset(sess, 0, sizeof(struct aesni_mb_session));
1237 memset(op->sym->session, 0,
1238 rte_cryptodev_sym_get_existing_header_session_size(
1240 rte_mempool_put(qp->sess_mp_priv, sess);
1241 rte_mempool_put(qp->sess_mp, op->sym->session);
1242 op->sym->session = NULL;
1249 post_process_mb_sync_job(JOB_AES_HMAC *job)
1253 st = job->user_data;
1254 st[0] = (job->status == STS_COMPLETED) ? 0 : EBADMSG;
1258 * Process a completed JOB_AES_HMAC job and keep processing jobs until
1259 * get_completed_job return NULL
1261 * @param qp Queue Pair to process
1262 * @param job JOB_AES_HMAC job
1265 * - Number of processed jobs
1268 handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
1269 struct rte_crypto_op **ops, uint16_t nb_ops)
1271 struct rte_crypto_op *op = NULL;
1272 unsigned processed_jobs = 0;
1274 while (job != NULL) {
1275 op = post_process_mb_job(qp, job);
1278 ops[processed_jobs++] = op;
1279 qp->stats.dequeued_count++;
1281 qp->stats.dequeue_err_count++;
1284 if (processed_jobs == nb_ops)
1287 job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
1290 return processed_jobs;
1293 static inline uint32_t
1294 handle_completed_sync_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr)
1298 for (i = 0; job != NULL; i++, job = IMB_GET_COMPLETED_JOB(mb_mgr))
1299 post_process_mb_sync_job(job);
1304 static inline uint32_t
1305 flush_mb_sync_mgr(MB_MGR *mb_mgr)
1309 job = IMB_FLUSH_JOB(mb_mgr);
1310 return handle_completed_sync_jobs(job, mb_mgr);
1313 static inline uint16_t
1314 flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
1317 int processed_ops = 0;
1319 /* Flush the remaining jobs */
1320 JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
1323 processed_ops += handle_completed_jobs(qp, job,
1324 &ops[processed_ops], nb_ops - processed_ops);
1326 return processed_ops;
1329 static inline JOB_AES_HMAC *
1330 set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
1332 job->chain_order = HASH_CIPHER;
1333 job->cipher_mode = NULL_CIPHER;
1334 job->hash_alg = NULL_HASH;
1335 job->cipher_direction = DECRYPT;
1337 /* Set user data to be crypto operation data struct */
1338 job->user_data = op;
1344 aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1347 struct aesni_mb_qp *qp = queue_pair;
1349 struct rte_crypto_op *op;
1352 int retval, processed_jobs = 0;
1354 if (unlikely(nb_ops == 0))
1357 uint8_t digest_idx = qp->digest_idx;
1359 /* Get next free mb job struct from mb manager */
1360 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1361 if (unlikely(job == NULL)) {
1362 /* if no free mb job structs we need to flush mb_mgr */
1363 processed_jobs += flush_mb_mgr(qp,
1364 &ops[processed_jobs],
1365 nb_ops - processed_jobs);
1367 if (nb_ops == processed_jobs)
1370 job = IMB_GET_NEXT_JOB(qp->mb_mgr);
1374 * Get next operation to process from ingress queue.
1375 * There is no need to return the job to the MB_MGR
1376 * if there are no more operations to process, since the MB_MGR
1377 * can use that pointer again in next get_next calls.
1379 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1383 retval = set_mb_job_params(job, qp, op, &digest_idx);
1384 if (unlikely(retval != 0)) {
1385 qp->stats.dequeue_err_count++;
1386 set_job_null_op(job, op);
1389 /* Submit job to multi-buffer for processing */
1390 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1391 job = IMB_SUBMIT_JOB(qp->mb_mgr);
1393 job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
1396 * If submit returns a processed job then handle it,
1397 * before submitting subsequent jobs
1400 processed_jobs += handle_completed_jobs(qp, job,
1401 &ops[processed_jobs],
1402 nb_ops - processed_jobs);
1404 } while (processed_jobs < nb_ops);
1406 qp->digest_idx = digest_idx;
1408 if (processed_jobs < 1)
1409 processed_jobs += flush_mb_mgr(qp,
1410 &ops[processed_jobs],
1411 nb_ops - processed_jobs);
1413 return processed_jobs;
1417 alloc_init_mb_mgr(enum aesni_mb_vector_mode vector_mode)
1419 MB_MGR *mb_mgr = alloc_mb_mgr(0);
1423 switch (vector_mode) {
1424 case RTE_AESNI_MB_SSE:
1425 init_mb_mgr_sse(mb_mgr);
1427 case RTE_AESNI_MB_AVX:
1428 init_mb_mgr_avx(mb_mgr);
1430 case RTE_AESNI_MB_AVX2:
1431 init_mb_mgr_avx2(mb_mgr);
1433 case RTE_AESNI_MB_AVX512:
1434 init_mb_mgr_avx512(mb_mgr);
1437 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
1438 free_mb_mgr(mb_mgr);
1446 aesni_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
1450 for (i = 0; i != vec->num; ++i)
1451 vec->status[i] = err;
1455 check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)
1457 /* no multi-seg support with current AESNI-MB PMD */
1460 else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)
1465 static inline JOB_AES_HMAC *
1466 submit_sync_job(MB_MGR *mb_mgr)
1468 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1469 return IMB_SUBMIT_JOB(mb_mgr);
1471 return IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
1475 static inline uint32_t
1476 generate_sync_dgst(struct rte_crypto_sym_vec *vec,
1477 const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1481 for (i = 0, k = 0; i != vec->num; i++) {
1482 if (vec->status[i] == 0) {
1483 memcpy(vec->digest[i], dgst[i], len);
1491 static inline uint32_t
1492 verify_sync_dgst(struct rte_crypto_sym_vec *vec,
1493 const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1497 for (i = 0, k = 0; i != vec->num; i++) {
1498 if (vec->status[i] == 0) {
1499 if (memcmp(vec->digest[i], dgst[i], len) != 0)
1500 vec->status[i] = EBADMSG;
1510 aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
1511 struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
1512 struct rte_crypto_sym_vec *vec)
1515 uint32_t i, j, k, len;
1519 struct aesni_mb_private *priv;
1520 struct aesni_mb_session *s;
1521 uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];
1523 s = get_sym_session_private_data(sess, dev->driver_id);
1525 aesni_mb_fill_error_code(vec, EINVAL);
1529 /* get per-thread MB MGR, create one if needed */
1530 mb_mgr = RTE_PER_LCORE(sync_mb_mgr);
1531 if (mb_mgr == NULL) {
1533 priv = dev->data->dev_private;
1534 mb_mgr = alloc_init_mb_mgr(priv->vector_mode);
1535 if (mb_mgr == NULL) {
1536 aesni_mb_fill_error_code(vec, ENOMEM);
1539 RTE_PER_LCORE(sync_mb_mgr) = mb_mgr;
1542 for (i = 0, j = 0, k = 0; i != vec->num; i++) {
1545 ret = check_crypto_sgl(sofs, vec->sgl + i);
1547 vec->status[i] = ret;
1551 buf = vec->sgl[i].vec[0].base;
1552 len = vec->sgl[i].vec[0].len;
1554 job = IMB_GET_NEXT_JOB(mb_mgr);
1556 k += flush_mb_sync_mgr(mb_mgr);
1557 job = IMB_GET_NEXT_JOB(mb_mgr);
1558 RTE_ASSERT(job != NULL);
1561 /* Submit job for processing */
1562 set_cpu_mb_job_params(job, s, sofs, buf, len,
1563 vec->iv[i], vec->aad[i], tmp_dgst[i],
1565 job = submit_sync_job(mb_mgr);
1568 /* handle completed jobs */
1569 k += handle_completed_sync_jobs(job, mb_mgr);
1572 /* flush remaining jobs */
1574 k += flush_mb_sync_mgr(mb_mgr);
1576 /* finish processing for successful jobs: check/update digest */
1578 if (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1579 k = verify_sync_dgst(vec,
1580 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1581 s->auth.req_digest_len);
1583 k = generate_sync_dgst(vec,
1584 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1585 s->auth.req_digest_len);
1591 static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
1594 vec_mode_to_flags(enum aesni_mb_vector_mode mode)
1597 case RTE_AESNI_MB_SSE:
1598 return RTE_CRYPTODEV_FF_CPU_SSE;
1599 case RTE_AESNI_MB_AVX:
1600 return RTE_CRYPTODEV_FF_CPU_AVX;
1601 case RTE_AESNI_MB_AVX2:
1602 return RTE_CRYPTODEV_FF_CPU_AVX2;
1603 case RTE_AESNI_MB_AVX512:
1604 return RTE_CRYPTODEV_FF_CPU_AVX512;
1606 AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", mode);
1612 cryptodev_aesni_mb_create(const char *name,
1613 struct rte_vdev_device *vdev,
1614 struct rte_cryptodev_pmd_init_params *init_params)
1616 struct rte_cryptodev *dev;
1617 struct aesni_mb_private *internals;
1618 enum aesni_mb_vector_mode vector_mode;
1621 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1623 AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
1627 /* Check CPU for supported vector instruction set */
1628 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
1629 vector_mode = RTE_AESNI_MB_AVX512;
1630 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
1631 vector_mode = RTE_AESNI_MB_AVX2;
1632 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
1633 vector_mode = RTE_AESNI_MB_AVX;
1635 vector_mode = RTE_AESNI_MB_SSE;
1637 dev->driver_id = cryptodev_driver_id;
1638 dev->dev_ops = rte_aesni_mb_pmd_ops;
1640 /* register rx/tx burst functions for data path */
1641 dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
1642 dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
1644 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1645 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1646 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
1647 RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
1648 RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
1650 /* Check CPU for support for AES instruction set */
1651 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
1652 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
1654 AESNI_MB_LOG(WARNING, "AES instructions not supported by CPU");
1656 dev->feature_flags |= vec_mode_to_flags(vector_mode);
1658 mb_mgr = alloc_init_mb_mgr(vector_mode);
1659 if (mb_mgr == NULL) {
1660 rte_cryptodev_pmd_destroy(dev);
1664 /* Set vector instructions mode supported */
1665 internals = dev->data->dev_private;
1667 internals->vector_mode = vector_mode;
1668 internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
1669 internals->mb_mgr = mb_mgr;
1671 AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
1672 imb_get_version_str());
1677 cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
1679 struct rte_cryptodev_pmd_init_params init_params = {
1681 sizeof(struct aesni_mb_private),
1683 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
1685 const char *name, *args;
1688 name = rte_vdev_device_name(vdev);
1692 args = rte_vdev_device_args(vdev);
1694 retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
1696 AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
1701 return cryptodev_aesni_mb_create(name, vdev, &init_params);
1705 cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
1707 struct rte_cryptodev *cryptodev;
1708 struct aesni_mb_private *internals;
1711 name = rte_vdev_device_name(vdev);
1715 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
1716 if (cryptodev == NULL)
1719 internals = cryptodev->data->dev_private;
1721 free_mb_mgr(internals->mb_mgr);
1722 if (RTE_PER_LCORE(sync_mb_mgr)) {
1723 free_mb_mgr(RTE_PER_LCORE(sync_mb_mgr));
1724 RTE_PER_LCORE(sync_mb_mgr) = NULL;
1727 return rte_cryptodev_pmd_destroy(cryptodev);
1730 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
1731 .probe = cryptodev_aesni_mb_probe,
1732 .remove = cryptodev_aesni_mb_remove
1735 static struct cryptodev_driver aesni_mb_crypto_drv;
1737 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
1738 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
1739 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
1740 "max_nb_queue_pairs=<int> "
1742 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
1743 cryptodev_aesni_mb_pmd_drv.driver,
1744 cryptodev_driver_id);
1745 RTE_LOG_REGISTER(aesni_mb_logtype_driver, pmd.crypto.aesni_mb, NOTICE);