1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2021 Intel Corporation
5 #include "pmd_aesni_mb_priv.h"
8 * Calculate the authentication pre-computes
10 * @param one_block_hash Function pointer
11 * to calculate digest on ipad/opad
12 * @param ipad Inner pad output byte array
13 * @param opad Outer pad output byte array
14 * @param hkey Authentication key
15 * @param hkey_len Authentication key length
16 * @param blocksize Block size of selected hash algo
19 calculate_auth_precomputes(hash_one_block_t one_block_hash,
20 uint8_t *ipad, uint8_t *opad,
21 const uint8_t *hkey, uint16_t hkey_len,
26 uint8_t ipad_buf[blocksize] __rte_aligned(16);
27 uint8_t opad_buf[blocksize] __rte_aligned(16);
29 /* Setup inner and outer pads */
30 memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
31 memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
33 /* XOR hash key with inner and outer pads */
34 length = hkey_len > blocksize ? blocksize : hkey_len;
36 for (i = 0; i < length; i++) {
37 ipad_buf[i] ^= hkey[i];
38 opad_buf[i] ^= hkey[i];
41 /* Compute partial hashes */
42 (*one_block_hash)(ipad_buf, ipad);
43 (*one_block_hash)(opad_buf, opad);
46 memset(ipad_buf, 0, blocksize);
47 memset(opad_buf, 0, blocksize);
51 is_aead_algo(IMB_HASH_ALG hash_alg, IMB_CIPHER_MODE cipher_mode)
53 return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 ||
54 hash_alg == IMB_AUTH_AES_CCM ||
55 (hash_alg == IMB_AUTH_AES_GMAC &&
56 cipher_mode == IMB_CIPHER_GCM));
59 /** Set session authentication parameters */
61 aesni_mb_set_session_auth_parameters(const IMB_MGR *mb_mgr,
62 struct aesni_mb_session *sess,
63 const struct rte_crypto_sym_xform *xform)
65 hash_one_block_t hash_oneblock_fn = NULL;
66 unsigned int key_larger_block_size = 0;
67 uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 };
68 uint32_t auth_precompute = 1;
71 sess->auth.algo = IMB_AUTH_NULL;
75 if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
76 IPSEC_MB_LOG(ERR, "Crypto xform struct not of type auth");
80 /* Set IV parameters */
81 sess->auth_iv.offset = xform->auth.iv.offset;
82 sess->auth_iv.length = xform->auth.iv.length;
84 /* Set the request digest size */
85 sess->auth.req_digest_len = xform->auth.digest_length;
87 /* Select auth generate/verify */
88 sess->auth.operation = xform->auth.op;
90 /* Set Authentication Parameters */
91 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
92 sess->auth.algo = IMB_AUTH_AES_XCBC;
94 uint16_t xcbc_mac_digest_len =
95 get_truncated_digest_byte_length(IMB_AUTH_AES_XCBC);
96 if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
97 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
100 sess->auth.gen_digest_len = sess->auth.req_digest_len;
102 IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
103 sess->auth.xcbc.k1_expanded,
104 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
108 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
111 sess->auth.algo = IMB_AUTH_AES_CMAC;
113 uint16_t cmac_digest_len =
114 get_digest_byte_length(IMB_AUTH_AES_CMAC);
116 if (sess->auth.req_digest_len > cmac_digest_len) {
117 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
121 * Multi-buffer lib supports digest sizes from 4 to 16 bytes
122 * in version 0.50 and sizes of 12 and 16 bytes,
124 * If size requested is different, generate the full digest
125 * (16 bytes) in a temporary location and then memcpy
126 * the requested number of bytes.
128 if (sess->auth.req_digest_len < 4)
129 sess->auth.gen_digest_len = cmac_digest_len;
131 sess->auth.gen_digest_len = sess->auth.req_digest_len;
133 IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
134 sess->auth.cmac.expkey, dust);
135 IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
136 sess->auth.cmac.skey1, sess->auth.cmac.skey2);
140 if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
141 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
142 sess->cipher.direction = IMB_DIR_ENCRYPT;
143 sess->chain_order = IMB_ORDER_CIPHER_HASH;
145 sess->cipher.direction = IMB_DIR_DECRYPT;
147 sess->auth.algo = IMB_AUTH_AES_GMAC;
148 if (sess->auth.req_digest_len >
149 get_digest_byte_length(IMB_AUTH_AES_GMAC)) {
150 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
153 sess->auth.gen_digest_len = sess->auth.req_digest_len;
154 sess->iv.length = xform->auth.iv.length;
155 sess->iv.offset = xform->auth.iv.offset;
157 switch (xform->auth.key.length) {
158 case IMB_KEY_128_BYTES:
159 IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data,
160 &sess->cipher.gcm_key);
161 sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
163 case IMB_KEY_192_BYTES:
164 IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data,
165 &sess->cipher.gcm_key);
166 sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
168 case IMB_KEY_256_BYTES:
169 IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data,
170 &sess->cipher.gcm_key);
171 sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
174 IPSEC_MB_LOG(ERR, "Invalid authentication key length\n");
181 if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
182 if (xform->auth.key.length == 16) {
183 sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;
184 } else if (xform->auth.key.length == 32) {
185 sess->auth.algo = IMB_AUTH_ZUC256_EIA3_BITLEN;
187 IPSEC_MB_LOG(ERR, "Invalid authentication key length\n");
191 uint16_t zuc_eia3_digest_len =
192 get_truncated_digest_byte_length(
193 IMB_AUTH_ZUC_EIA3_BITLEN);
194 if (sess->auth.req_digest_len != zuc_eia3_digest_len) {
195 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
198 sess->auth.gen_digest_len = sess->auth.req_digest_len;
200 memcpy(sess->auth.zuc_auth_key, xform->auth.key.data,
201 xform->auth.key.length);
203 } else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
204 sess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN;
205 uint16_t snow3g_uia2_digest_len =
206 get_truncated_digest_byte_length(
207 IMB_AUTH_SNOW3G_UIA2_BITLEN);
208 if (sess->auth.req_digest_len != snow3g_uia2_digest_len) {
209 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
212 sess->auth.gen_digest_len = sess->auth.req_digest_len;
214 IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->auth.key.data,
215 &sess->auth.pKeySched_snow3g_auth);
217 } else if (xform->auth.algo == RTE_CRYPTO_AUTH_KASUMI_F9) {
218 sess->auth.algo = IMB_AUTH_KASUMI_UIA1;
219 uint16_t kasumi_f9_digest_len =
220 get_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1);
221 if (sess->auth.req_digest_len != kasumi_f9_digest_len) {
222 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
225 sess->auth.gen_digest_len = sess->auth.req_digest_len;
227 IMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr, xform->auth.key.data,
228 &sess->auth.pKeySched_kasumi_auth);
232 switch (xform->auth.algo) {
233 case RTE_CRYPTO_AUTH_MD5_HMAC:
234 sess->auth.algo = IMB_AUTH_MD5;
235 hash_oneblock_fn = mb_mgr->md5_one_block;
237 case RTE_CRYPTO_AUTH_SHA1_HMAC:
238 sess->auth.algo = IMB_AUTH_HMAC_SHA_1;
239 hash_oneblock_fn = mb_mgr->sha1_one_block;
240 if (xform->auth.key.length > get_auth_algo_blocksize(
241 IMB_AUTH_HMAC_SHA_1)) {
243 xform->auth.key.data,
244 xform->auth.key.length,
246 key_larger_block_size = 1;
249 case RTE_CRYPTO_AUTH_SHA1:
250 sess->auth.algo = IMB_AUTH_SHA_1;
253 case RTE_CRYPTO_AUTH_SHA224_HMAC:
254 sess->auth.algo = IMB_AUTH_HMAC_SHA_224;
255 hash_oneblock_fn = mb_mgr->sha224_one_block;
256 if (xform->auth.key.length > get_auth_algo_blocksize(
257 IMB_AUTH_HMAC_SHA_224)) {
259 xform->auth.key.data,
260 xform->auth.key.length,
262 key_larger_block_size = 1;
265 case RTE_CRYPTO_AUTH_SHA224:
266 sess->auth.algo = IMB_AUTH_SHA_224;
269 case RTE_CRYPTO_AUTH_SHA256_HMAC:
270 sess->auth.algo = IMB_AUTH_HMAC_SHA_256;
271 hash_oneblock_fn = mb_mgr->sha256_one_block;
272 if (xform->auth.key.length > get_auth_algo_blocksize(
273 IMB_AUTH_HMAC_SHA_256)) {
275 xform->auth.key.data,
276 xform->auth.key.length,
278 key_larger_block_size = 1;
281 case RTE_CRYPTO_AUTH_SHA256:
282 sess->auth.algo = IMB_AUTH_SHA_256;
285 case RTE_CRYPTO_AUTH_SHA384_HMAC:
286 sess->auth.algo = IMB_AUTH_HMAC_SHA_384;
287 hash_oneblock_fn = mb_mgr->sha384_one_block;
288 if (xform->auth.key.length > get_auth_algo_blocksize(
289 IMB_AUTH_HMAC_SHA_384)) {
291 xform->auth.key.data,
292 xform->auth.key.length,
294 key_larger_block_size = 1;
297 case RTE_CRYPTO_AUTH_SHA384:
298 sess->auth.algo = IMB_AUTH_SHA_384;
301 case RTE_CRYPTO_AUTH_SHA512_HMAC:
302 sess->auth.algo = IMB_AUTH_HMAC_SHA_512;
303 hash_oneblock_fn = mb_mgr->sha512_one_block;
304 if (xform->auth.key.length > get_auth_algo_blocksize(
305 IMB_AUTH_HMAC_SHA_512)) {
307 xform->auth.key.data,
308 xform->auth.key.length,
310 key_larger_block_size = 1;
313 case RTE_CRYPTO_AUTH_SHA512:
314 sess->auth.algo = IMB_AUTH_SHA_512;
319 "Unsupported authentication algorithm selection");
322 uint16_t trunc_digest_size =
323 get_truncated_digest_byte_length(sess->auth.algo);
324 uint16_t full_digest_size =
325 get_digest_byte_length(sess->auth.algo);
327 if (sess->auth.req_digest_len > full_digest_size ||
328 sess->auth.req_digest_len == 0) {
329 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
333 if (sess->auth.req_digest_len != trunc_digest_size &&
334 sess->auth.req_digest_len != full_digest_size)
335 sess->auth.gen_digest_len = full_digest_size;
337 sess->auth.gen_digest_len = sess->auth.req_digest_len;
339 /* Plain SHA does not require precompute key */
340 if (auth_precompute == 0)
343 /* Calculate Authentication precomputes */
344 if (key_larger_block_size) {
345 calculate_auth_precomputes(hash_oneblock_fn,
346 sess->auth.pads.inner, sess->auth.pads.outer,
348 xform->auth.key.length,
349 get_auth_algo_blocksize(sess->auth.algo));
351 calculate_auth_precomputes(hash_oneblock_fn,
352 sess->auth.pads.inner, sess->auth.pads.outer,
353 xform->auth.key.data,
354 xform->auth.key.length,
355 get_auth_algo_blocksize(sess->auth.algo));
361 /** Set session cipher parameters */
363 aesni_mb_set_session_cipher_parameters(const IMB_MGR *mb_mgr,
364 struct aesni_mb_session *sess,
365 const struct rte_crypto_sym_xform *xform)
369 uint8_t is_docsis = 0;
371 uint8_t is_snow3g = 0;
372 uint8_t is_kasumi = 0;
375 sess->cipher.mode = IMB_CIPHER_NULL;
379 if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
380 IPSEC_MB_LOG(ERR, "Crypto xform struct not of type cipher");
384 /* Select cipher direction */
385 switch (xform->cipher.op) {
386 case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
387 sess->cipher.direction = IMB_DIR_ENCRYPT;
389 case RTE_CRYPTO_CIPHER_OP_DECRYPT:
390 sess->cipher.direction = IMB_DIR_DECRYPT;
393 IPSEC_MB_LOG(ERR, "Invalid cipher operation parameter");
397 /* Select cipher mode */
398 switch (xform->cipher.algo) {
399 case RTE_CRYPTO_CIPHER_AES_CBC:
400 sess->cipher.mode = IMB_CIPHER_CBC;
403 case RTE_CRYPTO_CIPHER_AES_CTR:
404 sess->cipher.mode = IMB_CIPHER_CNTR;
407 case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
408 sess->cipher.mode = IMB_CIPHER_DOCSIS_SEC_BPI;
411 case RTE_CRYPTO_CIPHER_DES_CBC:
412 sess->cipher.mode = IMB_CIPHER_DES;
414 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
415 sess->cipher.mode = IMB_CIPHER_DOCSIS_DES;
417 case RTE_CRYPTO_CIPHER_3DES_CBC:
418 sess->cipher.mode = IMB_CIPHER_DES3;
421 case RTE_CRYPTO_CIPHER_AES_ECB:
422 sess->cipher.mode = IMB_CIPHER_ECB;
425 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
426 sess->cipher.mode = IMB_CIPHER_ZUC_EEA3;
429 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
430 sess->cipher.mode = IMB_CIPHER_SNOW3G_UEA2_BITLEN;
433 case RTE_CRYPTO_CIPHER_KASUMI_F8:
434 sess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN;
438 IPSEC_MB_LOG(ERR, "Unsupported cipher mode parameter");
442 /* Set IV parameters */
443 sess->iv.offset = xform->cipher.iv.offset;
444 sess->iv.length = xform->cipher.iv.length;
446 /* Check key length and choose key expansion function for AES */
448 switch (xform->cipher.key.length) {
449 case IMB_KEY_128_BYTES:
450 sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
451 IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
452 sess->cipher.expanded_aes_keys.encode,
453 sess->cipher.expanded_aes_keys.decode);
455 case IMB_KEY_192_BYTES:
456 sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
457 IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
458 sess->cipher.expanded_aes_keys.encode,
459 sess->cipher.expanded_aes_keys.decode);
461 case IMB_KEY_256_BYTES:
462 sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
463 IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
464 sess->cipher.expanded_aes_keys.encode,
465 sess->cipher.expanded_aes_keys.decode);
468 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
471 } else if (is_docsis) {
472 switch (xform->cipher.key.length) {
473 case IMB_KEY_128_BYTES:
474 sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
475 IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
476 sess->cipher.expanded_aes_keys.encode,
477 sess->cipher.expanded_aes_keys.decode);
479 case IMB_KEY_256_BYTES:
480 sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
481 IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
482 sess->cipher.expanded_aes_keys.encode,
483 sess->cipher.expanded_aes_keys.decode);
486 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
489 } else if (is_3DES) {
490 uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
491 sess->cipher.exp_3des_keys.key[1],
492 sess->cipher.exp_3des_keys.key[2]};
494 switch (xform->cipher.key.length) {
496 IMB_DES_KEYSCHED(mb_mgr, keys[0],
497 xform->cipher.key.data);
498 IMB_DES_KEYSCHED(mb_mgr, keys[1],
499 xform->cipher.key.data + 8);
500 IMB_DES_KEYSCHED(mb_mgr, keys[2],
501 xform->cipher.key.data + 16);
503 /* Initialize keys - 24 bytes: [K1-K2-K3] */
504 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
505 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
506 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
509 IMB_DES_KEYSCHED(mb_mgr, keys[0],
510 xform->cipher.key.data);
511 IMB_DES_KEYSCHED(mb_mgr, keys[1],
512 xform->cipher.key.data + 8);
513 /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
514 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
515 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
516 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
519 IMB_DES_KEYSCHED(mb_mgr, keys[0],
520 xform->cipher.key.data);
522 /* Initialize keys - 8 bytes: [K1 = K2 = K3] */
523 sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
524 sess->cipher.exp_3des_keys.ks_ptr[1] = keys[0];
525 sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
528 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
532 sess->cipher.key_length_in_bytes = 24;
534 if (xform->cipher.key.length != 16 &&
535 xform->cipher.key.length != 32) {
536 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
539 sess->cipher.key_length_in_bytes = xform->cipher.key.length;
540 memcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data,
541 xform->cipher.key.length);
542 } else if (is_snow3g) {
543 if (xform->cipher.key.length != 16) {
544 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
547 sess->cipher.key_length_in_bytes = 16;
548 IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->cipher.key.data,
549 &sess->cipher.pKeySched_snow3g_cipher);
550 } else if (is_kasumi) {
551 if (xform->cipher.key.length != 16) {
552 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
555 sess->cipher.key_length_in_bytes = 16;
556 IMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data,
557 &sess->cipher.pKeySched_kasumi_cipher);
559 if (xform->cipher.key.length != 8) {
560 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
563 sess->cipher.key_length_in_bytes = 8;
565 IMB_DES_KEYSCHED(mb_mgr,
566 (uint64_t *)sess->cipher.expanded_aes_keys.encode,
567 xform->cipher.key.data);
568 IMB_DES_KEYSCHED(mb_mgr,
569 (uint64_t *)sess->cipher.expanded_aes_keys.decode,
570 xform->cipher.key.data);
577 aesni_mb_set_session_aead_parameters(const IMB_MGR *mb_mgr,
578 struct aesni_mb_session *sess,
579 const struct rte_crypto_sym_xform *xform)
581 switch (xform->aead.op) {
582 case RTE_CRYPTO_AEAD_OP_ENCRYPT:
583 sess->cipher.direction = IMB_DIR_ENCRYPT;
584 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
586 case RTE_CRYPTO_AEAD_OP_DECRYPT:
587 sess->cipher.direction = IMB_DIR_DECRYPT;
588 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
591 IPSEC_MB_LOG(ERR, "Invalid aead operation parameter");
595 /* Set IV parameters */
596 sess->iv.offset = xform->aead.iv.offset;
597 sess->iv.length = xform->aead.iv.length;
599 /* Set digest sizes */
600 sess->auth.req_digest_len = xform->aead.digest_length;
601 sess->auth.gen_digest_len = sess->auth.req_digest_len;
603 switch (xform->aead.algo) {
604 case RTE_CRYPTO_AEAD_AES_CCM:
605 sess->cipher.mode = IMB_CIPHER_CCM;
606 sess->auth.algo = IMB_AUTH_AES_CCM;
608 /* Check key length and choose key expansion function for AES */
609 switch (xform->aead.key.length) {
610 case IMB_KEY_128_BYTES:
611 sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
612 IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
613 sess->cipher.expanded_aes_keys.encode,
614 sess->cipher.expanded_aes_keys.decode);
616 case IMB_KEY_256_BYTES:
617 sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
618 IMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data,
619 sess->cipher.expanded_aes_keys.encode,
620 sess->cipher.expanded_aes_keys.decode);
623 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
627 /* CCM digests must be between 4 and 16 and an even number */
628 if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
629 sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
630 (sess->auth.req_digest_len & 1) == 1) {
631 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
636 case RTE_CRYPTO_AEAD_AES_GCM:
637 sess->cipher.mode = IMB_CIPHER_GCM;
638 sess->auth.algo = IMB_AUTH_AES_GMAC;
640 switch (xform->aead.key.length) {
641 case IMB_KEY_128_BYTES:
642 sess->cipher.key_length_in_bytes = IMB_KEY_128_BYTES;
643 IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
644 &sess->cipher.gcm_key);
646 case IMB_KEY_192_BYTES:
647 sess->cipher.key_length_in_bytes = IMB_KEY_192_BYTES;
648 IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
649 &sess->cipher.gcm_key);
651 case IMB_KEY_256_BYTES:
652 sess->cipher.key_length_in_bytes = IMB_KEY_256_BYTES;
653 IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
654 &sess->cipher.gcm_key);
657 IPSEC_MB_LOG(ERR, "Invalid cipher key length");
661 /* GCM digest size must be between 1 and 16 */
662 if (sess->auth.req_digest_len == 0 ||
663 sess->auth.req_digest_len > 16) {
664 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
669 case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
670 sess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305;
671 sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;
673 if (xform->aead.key.length != 32) {
674 IPSEC_MB_LOG(ERR, "Invalid key length");
677 sess->cipher.key_length_in_bytes = 32;
678 memcpy(sess->cipher.expanded_aes_keys.encode,
679 xform->aead.key.data, 32);
680 if (sess->auth.req_digest_len != 16) {
681 IPSEC_MB_LOG(ERR, "Invalid digest size\n");
686 IPSEC_MB_LOG(ERR, "Unsupported aead mode parameter");
693 /** Configure a aesni multi-buffer session from a crypto xform chain */
695 aesni_mb_session_configure(IMB_MGR *mb_mgr,
697 const struct rte_crypto_sym_xform *xform)
699 const struct rte_crypto_sym_xform *auth_xform = NULL;
700 const struct rte_crypto_sym_xform *cipher_xform = NULL;
701 const struct rte_crypto_sym_xform *aead_xform = NULL;
702 enum ipsec_mb_operation mode;
703 struct aesni_mb_session *sess = (struct aesni_mb_session *) priv_sess;
706 ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
707 &cipher_xform, &aead_xform);
711 /* Select Crypto operation - hash then cipher / cipher then hash */
713 case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
714 sess->chain_order = IMB_ORDER_HASH_CIPHER;
716 case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
717 case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
718 sess->chain_order = IMB_ORDER_CIPHER_HASH;
720 case IPSEC_MB_OP_HASH_GEN_ONLY:
721 case IPSEC_MB_OP_HASH_VERIFY_ONLY:
722 case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
723 sess->chain_order = IMB_ORDER_HASH_CIPHER;
726 * Multi buffer library operates only at two modes,
727 * IMB_ORDER_CIPHER_HASH and IMB_ORDER_HASH_CIPHER.
728 * When doing ciphering only, chain order depends
729 * on cipher operation: encryption is always
730 * the first operation and decryption the last one.
732 case IPSEC_MB_OP_ENCRYPT_ONLY:
733 sess->chain_order = IMB_ORDER_CIPHER_HASH;
735 case IPSEC_MB_OP_DECRYPT_ONLY:
736 sess->chain_order = IMB_ORDER_HASH_CIPHER;
738 case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
739 sess->chain_order = IMB_ORDER_CIPHER_HASH;
740 sess->aead.aad_len = xform->aead.aad_length;
742 case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
743 sess->chain_order = IMB_ORDER_HASH_CIPHER;
744 sess->aead.aad_len = xform->aead.aad_length;
746 case IPSEC_MB_OP_NOT_SUPPORTED:
749 "Unsupported operation chain order parameter");
753 /* Default IV length = 0 */
755 sess->auth_iv.length = 0;
757 ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
760 "Invalid/unsupported authentication parameters");
764 ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
767 IPSEC_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
772 ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
776 "Invalid/unsupported aead parameters");
784 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
785 /** Check DOCSIS security session configuration is valid */
787 check_docsis_sec_session(struct rte_security_session_conf *conf)
789 struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;
790 struct rte_security_docsis_xform *docsis = &conf->docsis;
792 /* Downlink: CRC generate -> Cipher encrypt */
793 if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
795 if (crypto_sym != NULL &&
796 crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
797 crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
798 crypto_sym->cipher.algo ==
799 RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
800 (crypto_sym->cipher.key.length == IMB_KEY_128_BYTES ||
801 crypto_sym->cipher.key.length == IMB_KEY_256_BYTES) &&
802 crypto_sym->cipher.iv.length == IMB_AES_BLOCK_SIZE &&
803 crypto_sym->next == NULL) {
806 /* Uplink: Cipher decrypt -> CRC verify */
807 } else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {
809 if (crypto_sym != NULL &&
810 crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
811 crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
812 crypto_sym->cipher.algo ==
813 RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
814 (crypto_sym->cipher.key.length == IMB_KEY_128_BYTES ||
815 crypto_sym->cipher.key.length == IMB_KEY_256_BYTES) &&
816 crypto_sym->cipher.iv.length == IMB_AES_BLOCK_SIZE &&
817 crypto_sym->next == NULL) {
825 /** Set DOCSIS security session auth (CRC) parameters */
827 aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
828 struct rte_security_docsis_xform *xform)
831 IPSEC_MB_LOG(ERR, "Invalid DOCSIS xform");
835 /* Select CRC generate/verify */
836 if (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) {
837 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
838 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
839 } else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
840 sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
841 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
843 IPSEC_MB_LOG(ERR, "Unsupported DOCSIS direction");
847 sess->auth.req_digest_len = RTE_ETHER_CRC_LEN;
848 sess->auth.gen_digest_len = RTE_ETHER_CRC_LEN;
854 * Parse DOCSIS security session configuration and set private session
858 aesni_mb_set_docsis_sec_session_parameters(
859 __rte_unused struct rte_cryptodev *dev,
860 struct rte_security_session_conf *conf,
863 IMB_MGR *mb_mgr = alloc_init_mb_mgr();
864 struct rte_security_docsis_xform *docsis_xform;
865 struct rte_crypto_sym_xform *cipher_xform;
866 struct aesni_mb_session *ipsec_sess = sess;
872 ret = check_docsis_sec_session(conf);
874 IPSEC_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
878 switch (conf->docsis.direction) {
879 case RTE_SECURITY_DOCSIS_UPLINK:
880 ipsec_sess->chain_order = IMB_ORDER_CIPHER_HASH;
881 docsis_xform = &conf->docsis;
882 cipher_xform = conf->crypto_xform;
884 case RTE_SECURITY_DOCSIS_DOWNLINK:
885 ipsec_sess->chain_order = IMB_ORDER_HASH_CIPHER;
886 cipher_xform = conf->crypto_xform;
887 docsis_xform = &conf->docsis;
890 IPSEC_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
895 /* Default IV length = 0 */
896 ipsec_sess->iv.length = 0;
898 ret = aesni_mb_set_docsis_sec_session_auth_parameters(ipsec_sess,
901 IPSEC_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
905 ret = aesni_mb_set_session_cipher_parameters(mb_mgr,
906 ipsec_sess, cipher_xform);
909 IPSEC_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
919 static inline uint64_t
920 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
923 struct rte_mbuf *m_src, *m_dst;
924 uint8_t *p_src, *p_dst;
925 uintptr_t u_src, u_dst;
926 uint32_t cipher_end, auth_end;
928 /* Only cipher then hash needs special calculation. */
929 if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH)
930 return op->sym->auth.data.offset;
932 m_src = op->sym->m_src;
933 m_dst = op->sym->m_dst;
935 p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
936 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
937 u_src = (uintptr_t)p_src;
938 u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
941 * Copy the content between cipher offset and auth offset for generating
944 if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
945 memcpy(p_dst + op->sym->auth.data.offset,
946 p_src + op->sym->auth.data.offset,
947 op->sym->cipher.data.offset -
948 op->sym->auth.data.offset);
951 * Copy the content between (cipher offset + length) and (auth offset +
952 * length) for generating correct digest
954 cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
955 auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
956 if (cipher_end < auth_end)
957 memcpy(p_dst + cipher_end, p_src + cipher_end,
958 auth_end - cipher_end);
961 * Since intel-ipsec-mb only supports positive values,
962 * we need to deduct the correct offset between src and dst.
965 return u_src < u_dst ? (u_dst - u_src) :
966 (UINT64_MAX - u_src + u_dst + 1);
970 set_cpu_mb_job_params(IMB_JOB *job, struct aesni_mb_session *session,
971 union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
972 struct rte_crypto_va_iova_ptr *iv,
973 struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)
975 /* Set crypto operation */
976 job->chain_order = session->chain_order;
978 /* Set cipher parameters */
979 job->cipher_direction = session->cipher.direction;
980 job->cipher_mode = session->cipher.mode;
982 job->key_len_in_bytes = session->cipher.key_length_in_bytes;
984 /* Set authentication parameters */
985 job->hash_alg = session->auth.algo;
988 switch (job->hash_alg) {
989 case IMB_AUTH_AES_XCBC:
990 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
991 job->u.XCBC._k2 = session->auth.xcbc.k2;
992 job->u.XCBC._k3 = session->auth.xcbc.k3;
994 job->enc_keys = session->cipher.expanded_aes_keys.encode;
995 job->dec_keys = session->cipher.expanded_aes_keys.decode;
998 case IMB_AUTH_AES_CCM:
999 job->u.CCM.aad = (uint8_t *)aad->va + 18;
1000 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1001 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1002 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1006 case IMB_AUTH_AES_CMAC:
1007 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1008 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1009 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1010 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1011 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1014 case IMB_AUTH_AES_GMAC:
1015 if (session->cipher.mode == IMB_CIPHER_GCM) {
1016 job->u.GCM.aad = aad->va;
1017 job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1020 job->u.GCM.aad = buf;
1021 job->u.GCM.aad_len_in_bytes = len;
1022 job->cipher_mode = IMB_CIPHER_GCM;
1024 job->enc_keys = &session->cipher.gcm_key;
1025 job->dec_keys = &session->cipher.gcm_key;
1028 case IMB_AUTH_CHACHA20_POLY1305:
1029 job->u.CHACHA20_POLY1305.aad = aad->va;
1030 job->u.CHACHA20_POLY1305.aad_len_in_bytes =
1031 session->aead.aad_len;
1032 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1033 job->dec_keys = session->cipher.expanded_aes_keys.encode;
1036 job->u.HMAC._hashed_auth_key_xor_ipad =
1037 session->auth.pads.inner;
1038 job->u.HMAC._hashed_auth_key_xor_opad =
1039 session->auth.pads.outer;
1041 if (job->cipher_mode == IMB_CIPHER_DES3) {
1042 job->enc_keys = session->cipher.exp_3des_keys.ks_ptr;
1043 job->dec_keys = session->cipher.exp_3des_keys.ks_ptr;
1045 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1046 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1051 * Multi-buffer library current only support returning a truncated
1052 * digest length as specified in the relevant IPsec RFCs
1055 /* Set digest location and length */
1056 job->auth_tag_output = digest;
1057 job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1059 /* Set IV parameters */
1060 job->iv_len_in_bytes = session->iv.length;
1062 /* Data Parameters */
1064 job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;
1065 job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;
1066 job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;
1067 if (job->hash_alg == IMB_AUTH_AES_GMAC &&
1068 session->cipher.mode != IMB_CIPHER_GCM) {
1069 job->msg_len_to_hash_in_bytes = 0;
1070 job->msg_len_to_cipher_in_bytes = 0;
1072 job->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -
1074 job->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -
1075 sofs.ofs.cipher.tail;
1078 job->user_data = udata;
1082 * Process a crypto operation and complete a IMB_JOB job structure for
1083 * submission to the multi buffer library for processing.
1085 * @param qp queue pair
1086 * @param job IMB_JOB structure to fill
1087 * @param op crypto op to process
1088 * @param digest_idx ID for digest to use
1091 * - 0 on success, the IMB_JOB will be filled
1092 * - -1 if invalid session, IMB_JOB will not be filled
1095 set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
1096 struct rte_crypto_op *op, uint8_t *digest_idx)
1098 struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
1099 struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
1100 struct aesni_mb_session *session;
1101 uint32_t m_offset, oop;
1103 session = ipsec_mb_get_session_private(qp, op);
1104 if (session == NULL) {
1105 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1109 /* Set crypto operation */
1110 job->chain_order = session->chain_order;
1112 /* Set cipher parameters */
1113 job->cipher_direction = session->cipher.direction;
1114 job->cipher_mode = session->cipher.mode;
1116 job->key_len_in_bytes = session->cipher.key_length_in_bytes;
1118 /* Set authentication parameters */
1119 job->hash_alg = session->auth.algo;
1121 const int aead = is_aead_algo(job->hash_alg, job->cipher_mode);
1123 switch (job->hash_alg) {
1124 case IMB_AUTH_AES_XCBC:
1125 job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
1126 job->u.XCBC._k2 = session->auth.xcbc.k2;
1127 job->u.XCBC._k3 = session->auth.xcbc.k3;
1129 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1130 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1133 case IMB_AUTH_AES_CCM:
1134 job->u.CCM.aad = op->sym->aead.aad.data + 18;
1135 job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
1136 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1137 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1140 case IMB_AUTH_AES_CMAC:
1141 job->u.CMAC._key_expanded = session->auth.cmac.expkey;
1142 job->u.CMAC._skey1 = session->auth.cmac.skey1;
1143 job->u.CMAC._skey2 = session->auth.cmac.skey2;
1144 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1145 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1148 case IMB_AUTH_AES_GMAC:
1149 if (session->cipher.mode == IMB_CIPHER_GCM) {
1150 job->u.GCM.aad = op->sym->aead.aad.data;
1151 job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
1154 job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
1155 uint8_t *, op->sym->auth.data.offset);
1156 job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
1157 job->cipher_mode = IMB_CIPHER_GCM;
1159 job->enc_keys = &session->cipher.gcm_key;
1160 job->dec_keys = &session->cipher.gcm_key;
1162 case IMB_AUTH_ZUC_EIA3_BITLEN:
1163 case IMB_AUTH_ZUC256_EIA3_BITLEN:
1164 job->u.ZUC_EIA3._key = session->auth.zuc_auth_key;
1165 job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1166 session->auth_iv.offset);
1168 case IMB_AUTH_SNOW3G_UIA2_BITLEN:
1169 job->u.SNOW3G_UIA2._key = (void *)
1170 &session->auth.pKeySched_snow3g_auth;
1171 job->u.SNOW3G_UIA2._iv =
1172 rte_crypto_op_ctod_offset(op, uint8_t *,
1173 session->auth_iv.offset);
1175 case IMB_AUTH_KASUMI_UIA1:
1176 job->u.KASUMI_UIA1._key = (void *)
1177 &session->auth.pKeySched_kasumi_auth;
1179 case IMB_AUTH_CHACHA20_POLY1305:
1180 job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
1181 job->u.CHACHA20_POLY1305.aad_len_in_bytes =
1182 session->aead.aad_len;
1183 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1184 job->dec_keys = session->cipher.expanded_aes_keys.encode;
1187 job->u.HMAC._hashed_auth_key_xor_ipad =
1188 session->auth.pads.inner;
1189 job->u.HMAC._hashed_auth_key_xor_opad =
1190 session->auth.pads.outer;
1192 if (job->cipher_mode == IMB_CIPHER_DES3) {
1193 job->enc_keys = session->cipher.exp_3des_keys.ks_ptr;
1194 job->dec_keys = session->cipher.exp_3des_keys.ks_ptr;
1196 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1197 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1202 m_offset = op->sym->aead.data.offset;
1204 m_offset = op->sym->cipher.data.offset;
1206 if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
1207 job->enc_keys = session->cipher.zuc_cipher_key;
1208 job->dec_keys = session->cipher.zuc_cipher_key;
1209 } else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
1210 job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
1212 } else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) {
1213 job->enc_keys = &session->cipher.pKeySched_kasumi_cipher;
1217 if (!op->sym->m_dst) {
1218 /* in-place operation */
1221 } else if (op->sym->m_dst == op->sym->m_src) {
1222 /* in-place operation */
1226 /* out-of-place operation */
1227 m_dst = op->sym->m_dst;
1231 /* Set digest output location */
1232 if (job->hash_alg != IMB_AUTH_NULL &&
1233 session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1234 job->auth_tag_output = qp_data->temp_digests[*digest_idx];
1235 *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
1238 job->auth_tag_output = op->sym->aead.digest.data;
1240 job->auth_tag_output = op->sym->auth.digest.data;
1242 if (session->auth.req_digest_len !=
1243 session->auth.gen_digest_len) {
1244 job->auth_tag_output =
1245 qp_data->temp_digests[*digest_idx];
1246 *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
1250 * Multi-buffer library current only support returning a truncated
1251 * digest length as specified in the relevant IPsec RFCs
1254 /* Set digest length */
1255 job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1257 /* Set IV parameters */
1258 job->iv_len_in_bytes = session->iv.length;
1260 /* Data Parameters */
1261 job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1262 job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
1264 switch (job->hash_alg) {
1265 case IMB_AUTH_AES_CCM:
1266 job->cipher_start_src_offset_in_bytes =
1267 op->sym->aead.data.offset;
1268 job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
1269 job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
1270 job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
1272 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1273 session->iv.offset + 1);
1276 case IMB_AUTH_AES_GMAC:
1277 if (session->cipher.mode == IMB_CIPHER_GCM) {
1278 job->cipher_start_src_offset_in_bytes =
1279 op->sym->aead.data.offset;
1280 job->hash_start_src_offset_in_bytes =
1281 op->sym->aead.data.offset;
1282 job->msg_len_to_cipher_in_bytes =
1283 op->sym->aead.data.length;
1284 job->msg_len_to_hash_in_bytes =
1285 op->sym->aead.data.length;
1287 job->cipher_start_src_offset_in_bytes =
1288 op->sym->auth.data.offset;
1289 job->hash_start_src_offset_in_bytes =
1290 op->sym->auth.data.offset;
1291 job->msg_len_to_cipher_in_bytes = 0;
1292 job->msg_len_to_hash_in_bytes = 0;
1295 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1296 session->iv.offset);
1299 case IMB_AUTH_CHACHA20_POLY1305:
1300 job->cipher_start_src_offset_in_bytes =
1301 op->sym->aead.data.offset;
1302 job->hash_start_src_offset_in_bytes =
1303 op->sym->aead.data.offset;
1304 job->msg_len_to_cipher_in_bytes =
1305 op->sym->aead.data.length;
1306 job->msg_len_to_hash_in_bytes =
1307 op->sym->aead.data.length;
1309 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1310 session->iv.offset);
1313 /* For SNOW3G, length and offsets are already in bits */
1314 job->cipher_start_src_offset_in_bytes =
1315 op->sym->cipher.data.offset;
1316 job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
1318 job->hash_start_src_offset_in_bytes = auth_start_offset(op,
1320 job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
1322 job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1323 session->iv.offset);
1326 if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
1327 job->msg_len_to_cipher_in_bytes >>= 3;
1328 else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)
1329 job->msg_len_to_hash_in_bytes >>= 3;
1331 /* Set user data to be crypto operation data struct */
1332 job->user_data = op;
1337 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1339 * Process a crypto operation containing a security op and complete a
1340 * IMB_JOB job structure for submission to the multi buffer library for
1344 set_sec_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
1345 struct rte_crypto_op *op, uint8_t *digest_idx)
1347 struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
1348 struct rte_mbuf *m_src, *m_dst;
1349 struct rte_crypto_sym_op *sym;
1350 struct aesni_mb_session *session = NULL;
1352 if (unlikely(op->sess_type != RTE_CRYPTO_OP_SECURITY_SESSION)) {
1353 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1356 session = (struct aesni_mb_session *)
1357 get_sec_session_private_data(op->sym->sec_session);
1359 if (unlikely(session == NULL)) {
1360 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1363 /* Only DOCSIS protocol operations supported now */
1364 if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||
1365 session->auth.algo != IMB_AUTH_DOCSIS_CRC32) {
1366 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1373 if (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {
1374 /* in-place operation */
1377 /* out-of-place operation not supported */
1378 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1382 /* Set crypto operation */
1383 job->chain_order = session->chain_order;
1385 /* Set cipher parameters */
1386 job->cipher_direction = session->cipher.direction;
1387 job->cipher_mode = session->cipher.mode;
1389 job->key_len_in_bytes = session->cipher.key_length_in_bytes;
1390 job->enc_keys = session->cipher.expanded_aes_keys.encode;
1391 job->dec_keys = session->cipher.expanded_aes_keys.decode;
1393 /* Set IV parameters */
1394 job->iv_len_in_bytes = session->iv.length;
1395 job->iv = (uint8_t *)op + session->iv.offset;
1397 /* Set authentication parameters */
1398 job->hash_alg = session->auth.algo;
1400 /* Set digest output location */
1401 job->auth_tag_output = qp_data->temp_digests[*digest_idx];
1402 *digest_idx = (*digest_idx + 1) % IMB_MAX_JOBS;
1404 /* Set digest length */
1405 job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
1407 /* Set data parameters */
1408 job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
1409 job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,
1410 sym->cipher.data.offset);
1412 job->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;
1413 job->msg_len_to_cipher_in_bytes = sym->cipher.data.length;
1415 job->hash_start_src_offset_in_bytes = sym->auth.data.offset;
1416 job->msg_len_to_hash_in_bytes = sym->auth.data.length;
1418 job->user_data = op;
1424 verify_docsis_sec_crc(IMB_JOB *job, uint8_t *status)
1426 uint16_t crc_offset;
1429 if (!job->msg_len_to_hash_in_bytes)
1432 crc_offset = job->hash_start_src_offset_in_bytes +
1433 job->msg_len_to_hash_in_bytes -
1434 job->cipher_start_src_offset_in_bytes;
1435 crc = job->dst + crc_offset;
1437 /* Verify CRC (at the end of the message) */
1438 if (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0)
1439 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1444 verify_digest(IMB_JOB *job, void *digest, uint16_t len, uint8_t *status)
1446 /* Verify digest if required */
1447 if (memcmp(job->auth_tag_output, digest, len) != 0)
1448 *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1452 generate_digest(IMB_JOB *job, struct rte_crypto_op *op,
1453 struct aesni_mb_session *sess)
1455 /* No extra copy needed */
1456 if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
1460 * This can only happen for HMAC, so only digest
1461 * for authentication algos is required
1463 memcpy(op->sym->auth.digest.data, job->auth_tag_output,
1464 sess->auth.req_digest_len);
1468 * Process a completed job and return rte_mbuf which job processed
1470 * @param qp Queue Pair to process
1471 * @param job IMB_JOB job to process
1474 * - Returns processed crypto operation.
1475 * - Returns NULL on invalid job
1477 static inline struct rte_crypto_op *
1478 post_process_mb_job(struct ipsec_mb_qp *qp, IMB_JOB *job)
1480 struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
1481 struct aesni_mb_session *sess = NULL;
1482 uint32_t driver_id = ipsec_mb_get_driver_id(
1483 IPSEC_MB_PMD_TYPE_AESNI_MB);
1485 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1486 uint8_t is_docsis_sec = 0;
1488 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1490 * Assuming at this point that if it's a security type op, that
1491 * this is for DOCSIS
1494 sess = get_sec_session_private_data(op->sym->sec_session);
1498 sess = get_sym_session_private_data(op->sym->session,
1502 if (unlikely(sess == NULL)) {
1503 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1507 if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
1508 switch (job->status) {
1509 case IMB_STATUS_COMPLETED:
1510 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1512 if (job->hash_alg == IMB_AUTH_NULL)
1515 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1516 if (is_aead_algo(job->hash_alg,
1519 op->sym->aead.digest.data,
1520 sess->auth.req_digest_len,
1522 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1523 else if (is_docsis_sec)
1524 verify_docsis_sec_crc(job,
1529 op->sym->auth.digest.data,
1530 sess->auth.req_digest_len,
1533 generate_digest(job, op, sess);
1536 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1540 /* Free session if a session-less crypto op */
1541 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1542 memset(sess, 0, sizeof(struct aesni_mb_session));
1543 memset(op->sym->session, 0,
1544 rte_cryptodev_sym_get_existing_header_session_size(
1546 rte_mempool_put(qp->sess_mp_priv, sess);
1547 rte_mempool_put(qp->sess_mp, op->sym->session);
1548 op->sym->session = NULL;
1555 post_process_mb_sync_job(IMB_JOB *job)
1559 st = job->user_data;
1560 st[0] = (job->status == IMB_STATUS_COMPLETED) ? 0 : EBADMSG;
1564 * Process a completed IMB_JOB job and keep processing jobs until
1565 * get_completed_job return NULL
1567 * @param qp Queue Pair to process
1568 * @param mb_mgr IMB_MGR to use
1569 * @param job IMB_JOB job
1570 * @param ops crypto ops to fill
1571 * @param nb_ops number of crypto ops
1574 * - Number of processed jobs
1577 handle_completed_jobs(struct ipsec_mb_qp *qp, IMB_MGR *mb_mgr,
1578 IMB_JOB *job, struct rte_crypto_op **ops,
1581 struct rte_crypto_op *op = NULL;
1582 uint16_t processed_jobs = 0;
1584 while (job != NULL) {
1585 op = post_process_mb_job(qp, job);
1588 ops[processed_jobs++] = op;
1589 qp->stats.dequeued_count++;
1591 qp->stats.dequeue_err_count++;
1594 if (processed_jobs == nb_ops)
1597 job = IMB_GET_COMPLETED_JOB(mb_mgr);
1600 return processed_jobs;
1603 static inline uint32_t
1604 handle_completed_sync_jobs(IMB_JOB *job, IMB_MGR *mb_mgr)
1608 for (i = 0; job != NULL; i++, job = IMB_GET_COMPLETED_JOB(mb_mgr))
1609 post_process_mb_sync_job(job);
1614 static inline uint32_t
1615 flush_mb_sync_mgr(IMB_MGR *mb_mgr)
1619 job = IMB_FLUSH_JOB(mb_mgr);
1620 return handle_completed_sync_jobs(job, mb_mgr);
1623 static inline uint16_t
1624 flush_mb_mgr(struct ipsec_mb_qp *qp, IMB_MGR *mb_mgr,
1625 struct rte_crypto_op **ops, uint16_t nb_ops)
1627 int processed_ops = 0;
1629 /* Flush the remaining jobs */
1630 IMB_JOB *job = IMB_FLUSH_JOB(mb_mgr);
1633 processed_ops += handle_completed_jobs(qp, mb_mgr, job,
1634 &ops[processed_ops], nb_ops - processed_ops);
1636 return processed_ops;
1639 static inline IMB_JOB *
1640 set_job_null_op(IMB_JOB *job, struct rte_crypto_op *op)
1642 job->chain_order = IMB_ORDER_HASH_CIPHER;
1643 job->cipher_mode = IMB_CIPHER_NULL;
1644 job->hash_alg = IMB_AUTH_NULL;
1645 job->cipher_direction = IMB_DIR_DECRYPT;
1647 /* Set user data to be crypto operation data struct */
1648 job->user_data = op;
1654 aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1657 struct ipsec_mb_qp *qp = queue_pair;
1658 IMB_MGR *mb_mgr = qp->mb_mgr;
1659 struct rte_crypto_op *op;
1661 int retval, processed_jobs = 0;
1663 if (unlikely(nb_ops == 0 || mb_mgr == NULL))
1666 uint8_t digest_idx = qp->digest_idx;
1669 /* Get next free mb job struct from mb manager */
1670 job = IMB_GET_NEXT_JOB(mb_mgr);
1671 if (unlikely(job == NULL)) {
1672 /* if no free mb job structs we need to flush mb_mgr */
1673 processed_jobs += flush_mb_mgr(qp, mb_mgr,
1674 &ops[processed_jobs],
1675 nb_ops - processed_jobs);
1677 if (nb_ops == processed_jobs)
1680 job = IMB_GET_NEXT_JOB(mb_mgr);
1684 * Get next operation to process from ingress queue.
1685 * There is no need to return the job to the IMB_MGR
1686 * if there are no more operations to process, since the IMB_MGR
1687 * can use that pointer again in next get_next calls.
1689 retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
1693 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1694 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1695 retval = set_sec_mb_job_params(job, qp, op,
1699 retval = set_mb_job_params(job, qp, op,
1702 if (unlikely(retval != 0)) {
1703 qp->stats.dequeue_err_count++;
1704 set_job_null_op(job, op);
1707 /* Submit job to multi-buffer for processing */
1708 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1709 job = IMB_SUBMIT_JOB(mb_mgr);
1711 job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
1714 * If submit returns a processed job then handle it,
1715 * before submitting subsequent jobs
1718 processed_jobs += handle_completed_jobs(qp, mb_mgr,
1719 job, &ops[processed_jobs],
1720 nb_ops - processed_jobs);
1722 } while (processed_jobs < nb_ops);
1724 qp->digest_idx = digest_idx;
1726 if (processed_jobs < 1)
1727 processed_jobs += flush_mb_mgr(qp, mb_mgr,
1728 &ops[processed_jobs],
1729 nb_ops - processed_jobs);
1731 return processed_jobs;
1736 ipsec_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
1740 for (i = 0; i != vec->num; ++i)
1741 vec->status[i] = err;
1745 check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)
1747 /* no multi-seg support with current AESNI-MB PMD */
1750 else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)
1755 static inline IMB_JOB *
1756 submit_sync_job(IMB_MGR *mb_mgr)
1758 #ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
1759 return IMB_SUBMIT_JOB(mb_mgr);
1761 return IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
1765 static inline uint32_t
1766 generate_sync_dgst(struct rte_crypto_sym_vec *vec,
1767 const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1771 for (i = 0, k = 0; i != vec->num; i++) {
1772 if (vec->status[i] == 0) {
1773 memcpy(vec->digest[i].va, dgst[i], len);
1781 static inline uint32_t
1782 verify_sync_dgst(struct rte_crypto_sym_vec *vec,
1783 const uint8_t dgst[][DIGEST_LENGTH_MAX], uint32_t len)
1787 for (i = 0, k = 0; i != vec->num; i++) {
1788 if (vec->status[i] == 0) {
1789 if (memcmp(vec->digest[i].va, dgst[i], len) != 0)
1790 vec->status[i] = EBADMSG;
1800 aesni_mb_process_bulk(struct rte_cryptodev *dev,
1801 struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
1802 struct rte_crypto_sym_vec *vec)
1805 uint32_t i, j, k, len;
1809 struct aesni_mb_session *s;
1810 uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];
1812 s = get_sym_session_private_data(sess, dev->driver_id);
1814 ipsec_mb_fill_error_code(vec, EINVAL);
1818 /* get per-thread MB MGR, create one if needed */
1819 mb_mgr = get_per_thread_mb_mgr();
1820 if (unlikely(mb_mgr == NULL))
1823 for (i = 0, j = 0, k = 0; i != vec->num; i++) {
1824 ret = check_crypto_sgl(sofs, vec->src_sgl + i);
1826 vec->status[i] = ret;
1830 buf = vec->src_sgl[i].vec[0].base;
1831 len = vec->src_sgl[i].vec[0].len;
1833 job = IMB_GET_NEXT_JOB(mb_mgr);
1835 k += flush_mb_sync_mgr(mb_mgr);
1836 job = IMB_GET_NEXT_JOB(mb_mgr);
1837 RTE_ASSERT(job != NULL);
1840 /* Submit job for processing */
1841 set_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i],
1842 &vec->aad[i], tmp_dgst[i], &vec->status[i]);
1843 job = submit_sync_job(mb_mgr);
1846 /* handle completed jobs */
1847 k += handle_completed_sync_jobs(job, mb_mgr);
1850 /* flush remaining jobs */
1852 k += flush_mb_sync_mgr(mb_mgr);
1854 /* finish processing for successful jobs: check/update digest */
1856 if (s->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1857 k = verify_sync_dgst(vec,
1858 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1859 s->auth.req_digest_len);
1861 k = generate_sync_dgst(vec,
1862 (const uint8_t (*)[DIGEST_LENGTH_MAX])tmp_dgst,
1863 s->auth.req_digest_len);
1869 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
1870 .dev_configure = ipsec_mb_config,
1871 .dev_start = ipsec_mb_start,
1872 .dev_stop = ipsec_mb_stop,
1873 .dev_close = ipsec_mb_close,
1875 .stats_get = ipsec_mb_stats_get,
1876 .stats_reset = ipsec_mb_stats_reset,
1878 .dev_infos_get = ipsec_mb_info_get,
1880 .queue_pair_setup = ipsec_mb_qp_setup,
1881 .queue_pair_release = ipsec_mb_qp_release,
1883 .sym_cpu_process = aesni_mb_process_bulk,
1885 .sym_session_get_size = ipsec_mb_sym_session_get_size,
1886 .sym_session_configure = ipsec_mb_sym_session_configure,
1887 .sym_session_clear = ipsec_mb_sym_session_clear
1890 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
1892 * Configure a aesni multi-buffer session from a security session
1896 aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
1897 struct rte_security_session *sess,
1898 struct rte_mempool *mempool)
1900 void *sess_private_data;
1901 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1904 if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
1905 conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
1906 IPSEC_MB_LOG(ERR, "Invalid security protocol");
1910 if (rte_mempool_get(mempool, &sess_private_data)) {
1911 IPSEC_MB_LOG(ERR, "Couldn't get object from session mempool");
1915 ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
1919 IPSEC_MB_LOG(ERR, "Failed to configure session parameters");
1921 /* Return session to mempool */
1922 rte_mempool_put(mempool, sess_private_data);
1926 set_sec_session_private_data(sess, sess_private_data);
1931 /** Clear the memory of session so it does not leave key material behind */
1933 aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
1934 struct rte_security_session *sess)
1936 void *sess_priv = get_sec_session_private_data(sess);
1939 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1941 memset(sess_priv, 0, sizeof(struct aesni_mb_session));
1942 set_sec_session_private_data(sess, NULL);
1943 rte_mempool_put(sess_mp, sess_priv);
1948 /** Get security capabilities for aesni multi-buffer */
1949 static const struct rte_security_capability *
1950 aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
1952 return aesni_mb_pmd_security_cap;
1955 static struct rte_security_ops aesni_mb_pmd_sec_ops = {
1956 .session_create = aesni_mb_pmd_sec_sess_create,
1957 .session_update = NULL,
1958 .session_stats_get = NULL,
1959 .session_destroy = aesni_mb_pmd_sec_sess_destroy,
1960 .set_pkt_metadata = NULL,
1961 .capabilities_get = aesni_mb_pmd_sec_capa_get
1964 struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
1967 aesni_mb_configure_dev(struct rte_cryptodev *dev)
1969 struct rte_security_ctx *security_instance;
1971 security_instance = rte_malloc("aesni_mb_sec",
1972 sizeof(struct rte_security_ctx),
1973 RTE_CACHE_LINE_SIZE);
1974 if (security_instance != NULL) {
1975 security_instance->device = (void *)dev;
1976 security_instance->ops = rte_aesni_mb_pmd_sec_ops;
1977 security_instance->sess_cnt = 0;
1978 dev->security_ctx = security_instance;
1989 aesni_mb_probe(struct rte_vdev_device *vdev)
1991 return ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_AESNI_MB);
1994 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
1995 .probe = aesni_mb_probe,
1996 .remove = ipsec_mb_remove
1999 static struct cryptodev_driver aesni_mb_crypto_drv;
2001 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD,
2002 cryptodev_aesni_mb_pmd_drv);
2003 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
2004 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
2005 "max_nb_queue_pairs=<int> socket_id=<int>");
2006 RTE_PMD_REGISTER_CRYPTO_DRIVER(
2007 aesni_mb_crypto_drv,
2008 cryptodev_aesni_mb_pmd_drv.driver,
2009 pmd_driver_id_aesni_mb);
2011 /* Constructor function to register aesni-mb PMD */
2012 RTE_INIT(ipsec_mb_register_aesni_mb)
2014 struct ipsec_mb_internals *aesni_mb_data =
2015 &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_AESNI_MB];
2017 aesni_mb_data->caps = aesni_mb_capabilities;
2018 aesni_mb_data->dequeue_burst = aesni_mb_dequeue_burst;
2019 aesni_mb_data->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2020 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2021 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
2022 RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
2023 RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
2024 RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
2026 aesni_mb_data->internals_priv_size = 0;
2027 aesni_mb_data->ops = &aesni_mb_pmd_ops;
2028 aesni_mb_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);
2029 aesni_mb_data->queue_pair_configure = NULL;
2030 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
2031 aesni_mb_data->security_ops = &aesni_mb_pmd_sec_ops;
2032 aesni_mb_data->dev_config = aesni_mb_configure_dev;
2033 aesni_mb_data->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
2035 aesni_mb_data->session_configure = aesni_mb_session_configure;
2036 aesni_mb_data->session_priv_size = sizeof(struct aesni_mb_session);