1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2017 Intel Corporation
5 #include <intel-ipsec-mb.h>
7 #include <rte_common.h>
8 #include <rte_hexdump.h>
9 #include <rte_cryptodev.h>
10 #include <rte_cryptodev_pmd.h>
11 #include <rte_bus_vdev.h>
12 #include <rte_malloc.h>
13 #include <rte_cpuflags.h>
15 #include "rte_aesni_mb_pmd_private.h"
17 #define AES_CCM_DIGEST_MIN_LEN 4
18 #define AES_CCM_DIGEST_MAX_LEN 16
19 #define HMAC_MAX_BLOCK_SIZE 128
20 static uint8_t cryptodev_driver_id;
22 typedef void (*hash_one_block_t)(const void *data, void *digest);
23 typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
26 * Calculate the authentication pre-computes
28 * @param one_block_hash Function pointer to calculate digest on ipad/opad
29 * @param ipad Inner pad output byte array
30 * @param opad Outer pad output byte array
31 * @param hkey Authentication key
32 * @param hkey_len Authentication key length
33 * @param blocksize Block size of selected hash algo
36 calculate_auth_precomputes(hash_one_block_t one_block_hash,
37 uint8_t *ipad, uint8_t *opad,
38 uint8_t *hkey, uint16_t hkey_len,
43 uint8_t ipad_buf[blocksize] __rte_aligned(16);
44 uint8_t opad_buf[blocksize] __rte_aligned(16);
46 /* Setup inner and outer pads */
47 memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
48 memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
50 /* XOR hash key with inner and outer pads */
51 length = hkey_len > blocksize ? blocksize : hkey_len;
53 for (i = 0; i < length; i++) {
54 ipad_buf[i] ^= hkey[i];
55 opad_buf[i] ^= hkey[i];
58 /* Compute partial hashes */
59 (*one_block_hash)(ipad_buf, ipad);
60 (*one_block_hash)(opad_buf, opad);
63 memset(ipad_buf, 0, blocksize);
64 memset(opad_buf, 0, blocksize);
67 /** Get xform chain order */
68 static enum aesni_mb_operation
69 aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
72 return AESNI_MB_OP_NOT_SUPPORTED;
74 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
75 if (xform->next == NULL)
76 return AESNI_MB_OP_CIPHER_ONLY;
77 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
78 return AESNI_MB_OP_CIPHER_HASH;
81 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
82 if (xform->next == NULL)
83 return AESNI_MB_OP_HASH_ONLY;
84 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
85 return AESNI_MB_OP_HASH_CIPHER;
88 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
89 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
90 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
91 return AESNI_MB_OP_AEAD_CIPHER_HASH;
93 return AESNI_MB_OP_AEAD_HASH_CIPHER;
97 return AESNI_MB_OP_NOT_SUPPORTED;
100 /** Set session authentication parameters */
102 aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
103 struct aesni_mb_session *sess,
104 const struct rte_crypto_sym_xform *xform)
106 hash_one_block_t hash_oneblock_fn;
107 unsigned int key_larger_block_size = 0;
108 uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
111 sess->auth.algo = NULL_HASH;
115 if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
116 AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
120 /* Set the request digest size */
121 sess->auth.req_digest_len = xform->auth.digest_length;
123 /* Select auth generate/verify */
124 sess->auth.operation = xform->auth.op;
126 /* Set Authentication Parameters */
127 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
128 sess->auth.algo = AES_XCBC;
130 uint16_t xcbc_mac_digest_len =
131 get_truncated_digest_byte_length(AES_XCBC);
132 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
133 AESNI_MB_LOG(ERR, "Invalid digest size\n");
136 sess->auth.gen_digest_len = sess->auth.req_digest_len;
137 (*mb_ops->aux.keyexp.aes_xcbc)(xform->auth.key.data,
138 sess->auth.xcbc.k1_expanded,
139 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
143 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
144 sess->auth.algo = AES_CMAC;
146 uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
148 if (sess->auth.req_digest_len > cmac_digest_len) {
149 AESNI_MB_LOG(ERR, "Invalid digest size\n");
153 * Multi-buffer lib supports digest sizes from 4 to 16 bytes
154 * in version 0.50 and sizes of 12 and 16 bytes,
156 * If size requested is different, generate the full digest
157 * (16 bytes) in a temporary location and then memcpy
158 * the requested number of bytes.
160 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
161 if (sess->auth.req_digest_len < 4)
163 uint16_t cmac_trunc_digest_len =
164 get_truncated_digest_byte_length(AES_CMAC);
165 if (sess->auth.req_digest_len != cmac_digest_len &&
166 sess->auth.req_digest_len != cmac_trunc_digest_len)
168 sess->auth.gen_digest_len = cmac_digest_len;
170 sess->auth.gen_digest_len = sess->auth.req_digest_len;
171 (*mb_ops->aux.keyexp.aes_cmac_expkey)(xform->auth.key.data,
172 sess->auth.cmac.expkey);
174 (*mb_ops->aux.keyexp.aes_cmac_subkey)(sess->auth.cmac.expkey,
175 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
179 switch (xform->auth.algo) {
180 case RTE_CRYPTO_AUTH_MD5_HMAC:
181 sess->auth.algo = MD5;
182 hash_oneblock_fn = mb_ops->aux.one_block.md5;
184 case RTE_CRYPTO_AUTH_SHA1_HMAC:
185 sess->auth.algo = SHA1;
186 hash_oneblock_fn = mb_ops->aux.one_block.sha1;
187 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
188 if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
189 mb_ops->aux.multi_block.sha1(
190 xform->auth.key.data,
191 xform->auth.key.length,
193 key_larger_block_size = 1;
197 case RTE_CRYPTO_AUTH_SHA224_HMAC:
198 sess->auth.algo = SHA_224;
199 hash_oneblock_fn = mb_ops->aux.one_block.sha224;
200 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
201 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
202 mb_ops->aux.multi_block.sha224(
203 xform->auth.key.data,
204 xform->auth.key.length,
206 key_larger_block_size = 1;
210 case RTE_CRYPTO_AUTH_SHA256_HMAC:
211 sess->auth.algo = SHA_256;
212 hash_oneblock_fn = mb_ops->aux.one_block.sha256;
213 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
214 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
215 mb_ops->aux.multi_block.sha256(
216 xform->auth.key.data,
217 xform->auth.key.length,
219 key_larger_block_size = 1;
223 case RTE_CRYPTO_AUTH_SHA384_HMAC:
224 sess->auth.algo = SHA_384;
225 hash_oneblock_fn = mb_ops->aux.one_block.sha384;
226 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
227 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
228 mb_ops->aux.multi_block.sha384(
229 xform->auth.key.data,
230 xform->auth.key.length,
232 key_larger_block_size = 1;
236 case RTE_CRYPTO_AUTH_SHA512_HMAC:
237 sess->auth.algo = SHA_512;
238 hash_oneblock_fn = mb_ops->aux.one_block.sha512;
239 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
240 if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
241 mb_ops->aux.multi_block.sha512(
242 xform->auth.key.data,
243 xform->auth.key.length,
245 key_larger_block_size = 1;
250 AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
253 uint16_t trunc_digest_size =
254 get_truncated_digest_byte_length(sess->auth.algo);
255 uint16_t full_digest_size =
256 get_digest_byte_length(sess->auth.algo);
258 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
259 if (sess->auth.req_digest_len > full_digest_size ||
260 sess->auth.req_digest_len == 0) {
262 if (sess->auth.req_digest_len != trunc_digest_size) {
264 AESNI_MB_LOG(ERR, "Invalid digest size\n");
268 if (sess->auth.req_digest_len != trunc_digest_size &&
269 sess->auth.req_digest_len != full_digest_size)
270 sess->auth.gen_digest_len = full_digest_size;
272 sess->auth.gen_digest_len = sess->auth.req_digest_len;
274 /* Calculate Authentication precomputes */
275 if (key_larger_block_size) {
276 calculate_auth_precomputes(hash_oneblock_fn,
277 sess->auth.pads.inner, sess->auth.pads.outer,
279 xform->auth.key.length,
280 get_auth_algo_blocksize(sess->auth.algo));
282 calculate_auth_precomputes(hash_oneblock_fn,
283 sess->auth.pads.inner, sess->auth.pads.outer,
284 xform->auth.key.data,
285 xform->auth.key.length,
286 get_auth_algo_blocksize(sess->auth.algo));
292 /** Set session cipher parameters */
294 aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
295 struct aesni_mb_session *sess,
296 const struct rte_crypto_sym_xform *xform)
300 aes_keyexp_t aes_keyexp_fn;
303 sess->cipher.mode = NULL_CIPHER;
307 if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
308 AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
312 /* Select cipher direction */
313 switch (xform->cipher.op) {
314 case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
315 sess->cipher.direction = ENCRYPT;
317 case RTE_CRYPTO_CIPHER_OP_DECRYPT:
318 sess->cipher.direction = DECRYPT;
321 AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
325 /* Select cipher mode */
326 switch (xform->cipher.algo) {
327 case RTE_CRYPTO_CIPHER_AES_CBC:
328 sess->cipher.mode = CBC;
331 case RTE_CRYPTO_CIPHER_AES_CTR:
332 sess->cipher.mode = CNTR;
335 case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
336 sess->cipher.mode = DOCSIS_SEC_BPI;
339 case RTE_CRYPTO_CIPHER_DES_CBC:
340 sess->cipher.mode = DES;
342 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
343 sess->cipher.mode = DOCSIS_DES;
345 case RTE_CRYPTO_CIPHER_3DES_CBC:
346 sess->cipher.mode = DES3;
350 AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
354 /* Set IV parameters */
355 sess->iv.offset = xform->cipher.iv.offset;
356 sess->iv.length = xform->cipher.iv.length;
358 /* Check key length and choose key expansion function for AES */
360 switch (xform->cipher.key.length) {
362 sess->cipher.key_length_in_bytes = AES_128_BYTES;
363 aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
366 sess->cipher.key_length_in_bytes = AES_192_BYTES;
367 aes_keyexp_fn = mb_ops->aux.keyexp.aes192;
370 sess->cipher.key_length_in_bytes = AES_256_BYTES;
371 aes_keyexp_fn = mb_ops->aux.keyexp.aes256;
374 AESNI_MB_LOG(ERR, "Invalid cipher key length");
378 /* Expanded cipher keys */
379 (*aes_keyexp_fn)(xform->cipher.key.data,
380 sess->cipher.expanded_aes_keys.encode,
381 sess->cipher.expanded_aes_keys.decode);
383 } else if (is_3DES) {
384 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
385 sess->cipher.exp_3des_keys.key[1],
386 sess->cipher.exp_3des_keys.key[2]};
388 switch (xform->cipher.key.length) {
390 des_key_schedule(keys[0], xform->cipher.key.data);
391 des_key_schedule(keys[1], xform->cipher.key.data+8);
392 des_key_schedule(keys[2], xform->cipher.key.data+16);
394 /* Initialize keys - 24 bytes: [K1-K2-K3] */
395 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
396 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
397 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
400 des_key_schedule(keys[0], xform->cipher.key.data);
401 des_key_schedule(keys[1], xform->cipher.key.data+8);
403 /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
404 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
405 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
406 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
409 des_key_schedule(keys[0], xform->cipher.key.data);
411 /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
412 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
413 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
414 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
417 AESNI_MB_LOG(ERR, "Invalid cipher key length");
421 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
422 sess->cipher.key_length_in_bytes = 24;
424 sess->cipher.key_length_in_bytes = 8;
427 if (xform->cipher.key.length != 8) {
428 AESNI_MB_LOG(ERR, "Invalid cipher key length");
431 sess->cipher.key_length_in_bytes = 8;
433 des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.encode,
434 xform->cipher.key.data);
435 des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.decode,
436 xform->cipher.key.data);
443 aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
444 struct aesni_mb_session *sess,
445 const struct rte_crypto_sym_xform *xform)
447 aes_keyexp_t aes_keyexp_fn;
449 switch (xform->aead.op) {
450 case RTE_CRYPTO_AEAD_OP_ENCRYPT:
451 sess->cipher.direction = ENCRYPT;
452 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
454 case RTE_CRYPTO_AEAD_OP_DECRYPT:
455 sess->cipher.direction = DECRYPT;
456 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
459 AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
463 switch (xform->aead.algo) {
464 case RTE_CRYPTO_AEAD_AES_CCM:
465 sess->cipher.mode = CCM;
466 sess->auth.algo = AES_CCM;
469 AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
473 /* Set IV parameters */
474 sess->iv.offset = xform->aead.iv.offset;
475 sess->iv.length = xform->aead.iv.length;
477 sess->auth.req_digest_len = xform->aead.digest_length;
478 /* CCM digests must be between 4 and 16 and an even number */
479 if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
480 sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
481 (sess->auth.req_digest_len & 1) == 1) {
482 AESNI_MB_LOG(ERR, "Invalid digest size\n");
485 sess->auth.gen_digest_len = sess->auth.req_digest_len;
487 /* Check key length and choose key expansion function for AES */
489 switch (xform->aead.key.length) {
491 sess->cipher.key_length_in_bytes = AES_128_BYTES;
492 aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
495 AESNI_MB_LOG(ERR, "Invalid cipher key length");
499 /* Expanded cipher keys */
500 (*aes_keyexp_fn)(xform->aead.key.data,
501 sess->cipher.expanded_aes_keys.encode,
502 sess->cipher.expanded_aes_keys.decode);
507 /** Parse crypto xform chain and set private session parameters */
509 aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
510 struct aesni_mb_session *sess,
511 const struct rte_crypto_sym_xform *xform)
513 const struct rte_crypto_sym_xform *auth_xform = NULL;
514 const struct rte_crypto_sym_xform *cipher_xform = NULL;
515 const struct rte_crypto_sym_xform *aead_xform = NULL;
518 /* Select Crypto operation - hash then cipher / cipher then hash */
519 switch (aesni_mb_get_chain_order(xform)) {
520 case AESNI_MB_OP_HASH_CIPHER:
521 sess->chain_order = HASH_CIPHER;
523 cipher_xform = xform->next;
525 case AESNI_MB_OP_CIPHER_HASH:
526 sess->chain_order = CIPHER_HASH;
527 auth_xform = xform->next;
528 cipher_xform = xform;
530 case AESNI_MB_OP_HASH_ONLY:
531 sess->chain_order = HASH_CIPHER;
535 case AESNI_MB_OP_CIPHER_ONLY:
537 * Multi buffer library operates only at two modes,
538 * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
539 * chain order depends on cipher operation: encryption is always
540 * the first operation and decryption the last one.
542 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
543 sess->chain_order = CIPHER_HASH;
545 sess->chain_order = HASH_CIPHER;
547 cipher_xform = xform;
549 case AESNI_MB_OP_AEAD_CIPHER_HASH:
550 sess->chain_order = CIPHER_HASH;
551 sess->aead.aad_len = xform->aead.aad_length;
554 case AESNI_MB_OP_AEAD_HASH_CIPHER:
555 sess->chain_order = HASH_CIPHER;
556 sess->aead.aad_len = xform->aead.aad_length;
559 case AESNI_MB_OP_NOT_SUPPORTED:
561 AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
565 /* Default IV length = 0 */
568 ret = aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform);
570 AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
574 ret = aesni_mb_set_session_cipher_parameters(mb_ops, sess,
577 AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
582 ret = aesni_mb_set_session_aead_parameters(mb_ops, sess,
585 AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
594 * burst enqueue, place crypto operations on ingress queue for processing.
596 * @param __qp Queue Pair to process
597 * @param ops Crypto operations for processing
598 * @param nb_ops Number of crypto operations for processing
601 * - Number of crypto operations enqueued
604 aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
607 struct aesni_mb_qp *qp = __qp;
609 unsigned int nb_enqueued;
611 nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
612 (void **)ops, nb_ops, NULL);
614 qp->stats.enqueued_count += nb_enqueued;
619 /** Get multi buffer session */
620 static inline struct aesni_mb_session *
621 get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
623 struct aesni_mb_session *sess = NULL;
625 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
626 if (likely(op->sym->session != NULL))
627 sess = (struct aesni_mb_session *)
628 get_sym_session_private_data(
630 cryptodev_driver_id);
633 void *_sess_private_data = NULL;
635 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
638 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
641 sess = (struct aesni_mb_session *)_sess_private_data;
643 if (unlikely(aesni_mb_set_session_parameters(qp->op_fns,
644 sess, op->sym->xform) != 0)) {
645 rte_mempool_put(qp->sess_mp, _sess);
646 rte_mempool_put(qp->sess_mp, _sess_private_data);
649 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
650 set_sym_session_private_data(op->sym->session,
651 cryptodev_driver_id, _sess_private_data);
654 if (unlikely(sess == NULL))
655 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
661 * Process a crypto operation and complete a JOB_AES_HMAC job structure for
662 * submission to the multi buffer library for processing.
664 * @param qp queue pair
665 * @param job JOB_AES_HMAC structure to fill
666 * @param m mbuf to process
669 * - Completed JOB_AES_HMAC structure pointer on success
670 * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
673 set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
674 struct rte_crypto_op *op, uint8_t *digest_idx)
676 struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
677 struct aesni_mb_session *session;
678 uint16_t m_offset = 0;
680 session = get_session(qp, op);
681 if (session == NULL) {
682 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
686 /* Set crypto operation */
687 job->chain_order = session->chain_order;
689 /* Set cipher parameters */
690 job->cipher_direction = session->cipher.direction;
691 job->cipher_mode = session->cipher.mode;
693 job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
695 if (job->cipher_mode == DES3) {
696 job->aes_enc_key_expanded =
697 session->cipher.exp_3des_keys.ks_ptr;
698 job->aes_dec_key_expanded =
699 session->cipher.exp_3des_keys.ks_ptr;
701 job->aes_enc_key_expanded =
702 session->cipher.expanded_aes_keys.encode;
703 job->aes_dec_key_expanded =
704 session->cipher.expanded_aes_keys.decode;
710 /* Set authentication parameters */
711 job->hash_alg = session->auth.algo;
712 if (job->hash_alg == AES_XCBC) {
713 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
714 job->u.XCBC._k2 = session->auth.xcbc.k2;
715 job->u.XCBC._k3 = session->auth.xcbc.k3;
716 } else if (job->hash_alg == AES_CCM) {
717 job->u.CCM.aad = op->sym->aead.aad.data + 18;
718 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
719 } else if (job->hash_alg == AES_CMAC) {
720 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
721 job->u.CMAC._skey1 = session->auth.cmac.skey1;
722 job->u.CMAC._skey2 = session->auth.cmac.skey2;
725 job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
726 job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
729 /* Mutable crypto operation parameters */
730 if (op->sym->m_dst) {
731 m_src = m_dst = op->sym->m_dst;
733 /* append space for output data to mbuf */
734 char *odata = rte_pktmbuf_append(m_dst,
735 rte_pktmbuf_data_len(op->sym->m_src));
737 AESNI_MB_LOG(ERR, "failed to allocate space in destination "
738 "mbuf for source data");
739 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
743 memcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*),
744 rte_pktmbuf_data_len(op->sym->m_src));
747 if (job->hash_alg == AES_CCM)
748 m_offset = op->sym->aead.data.offset;
750 m_offset = op->sym->cipher.data.offset;
753 /* Set digest output location */
754 if (job->hash_alg != NULL_HASH &&
755 session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
756 job->auth_tag_output = qp->temp_digests[*digest_idx];
757 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
759 if (job->hash_alg == AES_CCM)
760 job->auth_tag_output = op->sym->aead.digest.data;
762 job->auth_tag_output = op->sym->auth.digest.data;
764 if (session->auth.req_digest_len != session->auth.gen_digest_len) {
765 job->auth_tag_output = qp->temp_digests[*digest_idx];
766 *digest_idx = (*digest_idx + 1) % MAX_JOBS;
770 /* Set digest length */
771 job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
773 /* Set IV parameters */
774 job->iv_len_in_bytes = session->iv.length;
777 job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
778 job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
780 if (job->hash_alg == AES_CCM) {
781 job->cipher_start_src_offset_in_bytes =
782 op->sym->aead.data.offset;
783 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
784 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
785 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
787 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
788 session->iv.offset + 1);
790 job->cipher_start_src_offset_in_bytes =
791 op->sym->cipher.data.offset;
792 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
794 job->hash_start_src_offset_in_bytes = op->sym->auth.data.offset;
795 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
797 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
801 /* Set user data to be crypto operation data struct */
808 verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
809 struct aesni_mb_session *sess)
811 /* Verify digest if required */
812 if (job->hash_alg == AES_CCM) {
813 if (memcmp(job->auth_tag_output, op->sym->aead.digest.data,
814 sess->auth.req_digest_len) != 0)
815 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
817 if (memcmp(job->auth_tag_output, op->sym->auth.digest.data,
818 sess->auth.req_digest_len) != 0)
819 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
824 generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
825 struct aesni_mb_session *sess)
827 /* No extra copy neeed */
828 if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
832 * This can only happen for HMAC, so only digest
833 * for authentication algos is required
835 memcpy(op->sym->auth.digest.data, job->auth_tag_output,
836 sess->auth.req_digest_len);
840 * Process a completed job and return rte_mbuf which job processed
842 * @param qp Queue Pair to process
843 * @param job JOB_AES_HMAC job to process
846 * - Returns processed crypto operation.
847 * - Returns NULL on invalid job
849 static inline struct rte_crypto_op *
850 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
852 struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
853 struct aesni_mb_session *sess = get_sym_session_private_data(
855 cryptodev_driver_id);
857 if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
858 switch (job->status) {
860 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
862 if (job->hash_alg != NULL_HASH) {
863 if (sess->auth.operation ==
864 RTE_CRYPTO_AUTH_OP_VERIFY)
865 verify_digest(job, op, sess);
867 generate_digest(job, op, sess);
871 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
875 /* Free session if a session-less crypto op */
876 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
877 memset(sess, 0, sizeof(struct aesni_mb_session));
878 memset(op->sym->session, 0,
879 rte_cryptodev_sym_get_header_session_size());
880 rte_mempool_put(qp->sess_mp, sess);
881 rte_mempool_put(qp->sess_mp, op->sym->session);
882 op->sym->session = NULL;
889 * Process a completed JOB_AES_HMAC job and keep processing jobs until
890 * get_completed_job return NULL
892 * @param qp Queue Pair to process
893 * @param job JOB_AES_HMAC job
896 * - Number of processed jobs
899 handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
900 struct rte_crypto_op **ops, uint16_t nb_ops)
902 struct rte_crypto_op *op = NULL;
903 unsigned processed_jobs = 0;
905 while (job != NULL) {
906 op = post_process_mb_job(qp, job);
909 ops[processed_jobs++] = op;
910 qp->stats.dequeued_count++;
912 qp->stats.dequeue_err_count++;
915 if (processed_jobs == nb_ops)
918 job = (*qp->op_fns->job.get_completed_job)(qp->mb_mgr);
921 return processed_jobs;
924 static inline uint16_t
925 flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
928 int processed_ops = 0;
930 /* Flush the remaining jobs */
931 JOB_AES_HMAC *job = (*qp->op_fns->job.flush_job)(qp->mb_mgr);
934 processed_ops += handle_completed_jobs(qp, job,
935 &ops[processed_ops], nb_ops - processed_ops);
937 return processed_ops;
940 static inline JOB_AES_HMAC *
941 set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
943 job->chain_order = HASH_CIPHER;
944 job->cipher_mode = NULL_CIPHER;
945 job->hash_alg = NULL_HASH;
946 job->cipher_direction = DECRYPT;
948 /* Set user data to be crypto operation data struct */
955 aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
958 struct aesni_mb_qp *qp = queue_pair;
960 struct rte_crypto_op *op;
963 int retval, processed_jobs = 0;
965 if (unlikely(nb_ops == 0))
968 uint8_t digest_idx = qp->digest_idx;
970 /* Get next free mb job struct from mb manager */
971 job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
972 if (unlikely(job == NULL)) {
973 /* if no free mb job structs we need to flush mb_mgr */
974 processed_jobs += flush_mb_mgr(qp,
975 &ops[processed_jobs],
976 nb_ops - processed_jobs);
978 if (nb_ops == processed_jobs)
981 job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
985 * Get next operation to process from ingress queue.
986 * There is no need to return the job to the MB_MGR
987 * if there are no more operations to process, since the MB_MGR
988 * can use that pointer again in next get_next calls.
990 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
994 retval = set_mb_job_params(job, qp, op, &digest_idx);
995 if (unlikely(retval != 0)) {
996 qp->stats.dequeue_err_count++;
997 set_job_null_op(job, op);
1000 /* Submit job to multi-buffer for processing */
1001 job = (*qp->op_fns->job.submit)(qp->mb_mgr);
1004 * If submit returns a processed job then handle it,
1005 * before submitting subsequent jobs
1008 processed_jobs += handle_completed_jobs(qp, job,
1009 &ops[processed_jobs],
1010 nb_ops - processed_jobs);
1012 } while (processed_jobs < nb_ops);
1014 qp->digest_idx = digest_idx;
1016 if (processed_jobs < 1)
1017 processed_jobs += flush_mb_mgr(qp,
1018 &ops[processed_jobs],
1019 nb_ops - processed_jobs);
1021 return processed_jobs;
1024 static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
1027 cryptodev_aesni_mb_create(const char *name,
1028 struct rte_vdev_device *vdev,
1029 struct rte_cryptodev_pmd_init_params *init_params)
1031 struct rte_cryptodev *dev;
1032 struct aesni_mb_private *internals;
1033 enum aesni_mb_vector_mode vector_mode;
1035 /* Check CPU for support for AES instruction set */
1036 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
1037 AESNI_MB_LOG(ERR, "AES instructions not supported by CPU");
1041 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1043 AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
1047 /* Check CPU for supported vector instruction set */
1048 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
1049 vector_mode = RTE_AESNI_MB_AVX512;
1050 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
1051 vector_mode = RTE_AESNI_MB_AVX2;
1052 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
1053 vector_mode = RTE_AESNI_MB_AVX;
1055 vector_mode = RTE_AESNI_MB_SSE;
1057 dev->driver_id = cryptodev_driver_id;
1058 dev->dev_ops = rte_aesni_mb_pmd_ops;
1060 /* register rx/tx burst functions for data path */
1061 dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
1062 dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
1064 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1065 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1066 RTE_CRYPTODEV_FF_CPU_AESNI;
1068 switch (vector_mode) {
1069 case RTE_AESNI_MB_SSE:
1070 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
1072 case RTE_AESNI_MB_AVX:
1073 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
1075 case RTE_AESNI_MB_AVX2:
1076 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
1078 case RTE_AESNI_MB_AVX512:
1079 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
1085 /* Set vector instructions mode supported */
1086 internals = dev->data->dev_private;
1088 internals->vector_mode = vector_mode;
1089 internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
1091 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
1092 AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
1093 imb_get_version_str());
1095 AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: 0.49.0\n");
1102 cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
1104 struct rte_cryptodev_pmd_init_params init_params = {
1106 sizeof(struct aesni_mb_private),
1108 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
1110 const char *name, *args;
1113 name = rte_vdev_device_name(vdev);
1117 args = rte_vdev_device_args(vdev);
1119 retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
1121 AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
1126 return cryptodev_aesni_mb_create(name, vdev, &init_params);
1130 cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
1132 struct rte_cryptodev *cryptodev;
1135 name = rte_vdev_device_name(vdev);
1139 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
1140 if (cryptodev == NULL)
1143 return rte_cryptodev_pmd_destroy(cryptodev);
1146 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
1147 .probe = cryptodev_aesni_mb_probe,
1148 .remove = cryptodev_aesni_mb_remove
1151 static struct cryptodev_driver aesni_mb_crypto_drv;
1153 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
1154 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
1155 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
1156 "max_nb_queue_pairs=<int> "
1158 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
1159 cryptodev_aesni_mb_pmd_drv.driver,
1160 cryptodev_driver_id);
1162 RTE_INIT(aesni_mb_init_log)
1164 aesni_mb_logtype_driver = rte_log_register("pmd.crypto.aesni_mb");