4 * Copyright (C) Cavium, Inc. 2017.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Cavium, Inc nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #include <rte_common.h>
36 #include <rte_hexdump.h>
37 #include <rte_cryptodev.h>
38 #include <rte_cryptodev_pmd.h>
39 #include <rte_cryptodev_vdev.h>
41 #include <rte_malloc.h>
42 #include <rte_cpuflags.h>
44 #include "armv8_crypto_defs.h"
46 #include "rte_armv8_pmd_private.h"
48 static uint8_t cryptodev_driver_id;
50 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev);
53 * Pointers to the supported combined mode crypto functions are stored
54 * in the static tables. Each combined (chained) cryptographic operation
55 * can be described by a set of numbers:
56 * - order: order of operations (cipher, auth) or (auth, cipher)
57 * - direction: encryption or decryption
58 * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc.
59 * - aalg: authentication algorithm such as SHA1, SHA256, etc.
60 * - keyl: cipher key length, for example 128, 192, 256 bits
62 * In order to quickly acquire each function pointer based on those numbers,
63 * a hierarchy of arrays is maintained. The final level, 3D array is indexed
64 * by the combined mode function parameters only (cipher algorithm,
65 * authentication algorithm and key length).
67 * This gives 3 memory accesses to obtain a function pointer instead of
68 * traversing the array manually and comparing function parameters on each loop.
82 * 3D array type for ARM Combined Mode crypto functions pointers.
83 * CRYPTO_CIPHER_MAX: max cipher ID number
84 * CRYPTO_AUTH_MAX: max auth ID number
85 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number
87 typedef const crypto_func_t
88 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
90 /* Evaluate to key length definition */
91 #define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl)
93 /* Local aliases for supported ciphers */
94 #define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC
95 /* Local aliases for supported hashes */
96 #define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC
97 #define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC
100 * Arrays containing pointers to particular cryptographic,
101 * combined mode functions.
102 * crypto_op_ca_encrypt: cipher (encrypt), authenticate
103 * crypto_op_ca_decrypt: cipher (decrypt), authenticate
104 * crypto_op_ac_encrypt: authenticate, cipher (encrypt)
105 * crypto_op_ac_decrypt: authenticate, cipher (decrypt)
107 static const crypto_func_tbl_t
108 crypto_op_ca_encrypt = {
109 /* [cipher alg][auth alg][key length] = crypto_function, */
110 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac,
111 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac,
114 static const crypto_func_tbl_t
115 crypto_op_ca_decrypt = {
119 static const crypto_func_tbl_t
120 crypto_op_ac_encrypt = {
124 static const crypto_func_tbl_t
125 crypto_op_ac_decrypt = {
126 /* [cipher alg][auth alg][key length] = crypto_function, */
127 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec,
128 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec,
132 * Arrays containing pointers to particular cryptographic function sets,
133 * covering given cipher operation directions (encrypt, decrypt)
134 * for each order of cipher and authentication pairs.
136 static const crypto_func_tbl_t *
137 crypto_cipher_auth[] = {
138 &crypto_op_ca_encrypt,
139 &crypto_op_ca_decrypt,
143 static const crypto_func_tbl_t *
144 crypto_auth_cipher[] = {
145 &crypto_op_ac_encrypt,
146 &crypto_op_ac_decrypt,
151 * Top level array containing pointers to particular cryptographic
152 * function sets, covering given order of chained operations.
153 * crypto_cipher_auth: cipher first, authenticate after
154 * crypto_auth_cipher: authenticate first, cipher after
156 static const crypto_func_tbl_t **
157 crypto_chain_order[] = {
164 * Extract particular combined mode crypto function from the 3D array.
166 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \
168 crypto_func_tbl_t *func_tbl = \
169 (crypto_chain_order[(order)])[(cop)]; \
171 ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \
174 /*----------------------------------------------------------------------------*/
177 * 2D array type for ARM key schedule functions pointers.
178 * CRYPTO_CIPHER_MAX: max cipher ID number
179 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number
181 typedef const crypto_key_sched_t
182 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
184 static const crypto_key_sched_tbl_t
185 crypto_key_sched_encrypt = {
186 /* [cipher alg][key length] = key_expand_func, */
187 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc,
190 static const crypto_key_sched_tbl_t
191 crypto_key_sched_decrypt = {
192 /* [cipher alg][key length] = key_expand_func, */
193 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec,
197 * Top level array containing pointers to particular key generation
198 * function sets, covering given operation direction.
199 * crypto_key_sched_encrypt: keys for encryption
200 * crypto_key_sched_decrypt: keys for decryption
202 static const crypto_key_sched_tbl_t *
203 crypto_key_sched_dir[] = {
204 &crypto_key_sched_encrypt,
205 &crypto_key_sched_decrypt,
210 * Extract particular combined mode crypto function from the 3D array.
212 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \
214 crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \
216 ((*ks_tbl)[(calg)][KEYL(keyl)]); \
219 /*----------------------------------------------------------------------------*/
222 *------------------------------------------------------------------------------
224 *------------------------------------------------------------------------------
227 /** Get xform chain order */
228 static enum armv8_crypto_chain_order
229 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform)
233 * This driver currently covers only chained operations.
234 * Ignore only cipher or only authentication operations
235 * or chains longer than 2 xform structures.
237 if (xform->next == NULL || xform->next->next != NULL)
238 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
240 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
241 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
242 return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER;
245 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
246 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
247 return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH;
250 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
254 auth_hmac_pad_prepare(struct armv8_crypto_session *sess,
255 const struct rte_crypto_sym_xform *xform)
259 /* Generate i_key_pad and o_key_pad */
260 memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad));
261 rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key,
262 xform->auth.key.length);
263 memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad));
264 rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key,
265 xform->auth.key.length);
267 * XOR key with IPAD/OPAD values to obtain i_key_pad
269 * Byte-by-byte operation may seem to be the less efficient
270 * here but in fact it's the opposite.
271 * The result ASM code is likely operate on NEON registers
272 * (load auth key to Qx, load IPAD/OPAD to multiple
273 * elements of Qy, eor 128 bits at once).
275 for (i = 0; i < SHA_BLOCK_MAX; i++) {
276 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE;
277 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE;
282 auth_set_prerequisites(struct armv8_crypto_session *sess,
283 const struct rte_crypto_sym_xform *xform)
285 uint8_t partial[64] = { 0 };
288 switch (xform->auth.algo) {
289 case RTE_CRYPTO_AUTH_SHA1_HMAC:
291 * Generate authentication key, i_key_pad and o_key_pad.
293 /* Zero memory under key */
294 memset(sess->auth.hmac.key, 0, SHA1_AUTH_KEY_LENGTH);
296 if (xform->auth.key.length > SHA1_AUTH_KEY_LENGTH) {
298 * In case the key is longer than 160 bits
299 * the algorithm will use SHA1(key) instead.
301 error = sha1_block(NULL, xform->auth.key.data,
302 sess->auth.hmac.key, xform->auth.key.length);
307 * Now copy the given authentication key to the session
308 * key assuming that the session key is zeroed there is
309 * no need for additional zero padding if the key is
310 * shorter than SHA1_AUTH_KEY_LENGTH.
312 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
313 xform->auth.key.length);
316 /* Prepare HMAC padding: key|pattern */
317 auth_hmac_pad_prepare(sess, xform);
319 * Calculate partial hash values for i_key_pad and o_key_pad.
320 * Will be used as initialization state for final HMAC.
322 error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad,
323 partial, SHA1_BLOCK_SIZE);
326 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE);
328 error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad,
329 partial, SHA1_BLOCK_SIZE);
332 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE);
335 case RTE_CRYPTO_AUTH_SHA256_HMAC:
337 * Generate authentication key, i_key_pad and o_key_pad.
339 /* Zero memory under key */
340 memset(sess->auth.hmac.key, 0, SHA256_AUTH_KEY_LENGTH);
342 if (xform->auth.key.length > SHA256_AUTH_KEY_LENGTH) {
344 * In case the key is longer than 256 bits
345 * the algorithm will use SHA256(key) instead.
347 error = sha256_block(NULL, xform->auth.key.data,
348 sess->auth.hmac.key, xform->auth.key.length);
353 * Now copy the given authentication key to the session
354 * key assuming that the session key is zeroed there is
355 * no need for additional zero padding if the key is
356 * shorter than SHA256_AUTH_KEY_LENGTH.
358 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
359 xform->auth.key.length);
362 /* Prepare HMAC padding: key|pattern */
363 auth_hmac_pad_prepare(sess, xform);
365 * Calculate partial hash values for i_key_pad and o_key_pad.
366 * Will be used as initialization state for final HMAC.
368 error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad,
369 partial, SHA256_BLOCK_SIZE);
372 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE);
374 error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad,
375 partial, SHA256_BLOCK_SIZE);
378 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE);
389 cipher_set_prerequisites(struct armv8_crypto_session *sess,
390 const struct rte_crypto_sym_xform *xform)
392 crypto_key_sched_t cipher_key_sched;
394 cipher_key_sched = sess->cipher.key_sched;
395 if (likely(cipher_key_sched != NULL)) {
396 /* Set up cipher session key */
397 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data);
404 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess,
405 const struct rte_crypto_sym_xform *cipher_xform,
406 const struct rte_crypto_sym_xform *auth_xform)
408 enum armv8_crypto_chain_order order;
409 enum armv8_crypto_cipher_operation cop;
410 enum rte_crypto_cipher_algorithm calg;
411 enum rte_crypto_auth_algorithm aalg;
413 /* Validate and prepare scratch order of combined operations */
414 switch (sess->chain_order) {
415 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
416 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
417 order = sess->chain_order;
422 /* Select cipher direction */
423 sess->cipher.direction = cipher_xform->cipher.op;
424 /* Select cipher key */
425 sess->cipher.key.length = cipher_xform->cipher.key.length;
426 /* Set cipher direction */
427 cop = sess->cipher.direction;
428 /* Set cipher algorithm */
429 calg = cipher_xform->cipher.algo;
431 /* Select cipher algo */
433 /* Cover supported cipher algorithms */
434 case RTE_CRYPTO_CIPHER_AES_CBC:
435 sess->cipher.algo = calg;
436 /* IV len is always 16 bytes (block size) for AES CBC */
437 sess->cipher.iv.length = 16;
442 /* Select auth generate/verify */
443 sess->auth.operation = auth_xform->auth.op;
445 /* Select auth algo */
446 switch (auth_xform->auth.algo) {
447 /* Cover supported hash algorithms */
448 case RTE_CRYPTO_AUTH_SHA1_HMAC:
449 case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */
450 aalg = auth_xform->auth.algo;
451 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC;
457 /* Set the digest length */
458 sess->auth.digest_length = auth_xform->auth.digest_length;
460 /* Verify supported key lengths and extract proper algorithm */
461 switch (cipher_xform->cipher.key.length << 3) {
464 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128);
465 sess->cipher.key_sched =
466 CRYPTO_GET_KEY_SCHED(cop, calg, 128);
470 /* These key lengths are not supported yet */
471 default: /* Fall through */
472 sess->crypto_func = NULL;
473 sess->cipher.key_sched = NULL;
477 if (unlikely(sess->crypto_func == NULL)) {
479 * If we got here that means that there must be a bug
480 * in the algorithms selection above. Nevertheless keep
481 * it here to catch bug immediately and avoid NULL pointer
482 * dereference in OPs processing.
484 ARMV8_CRYPTO_LOG_ERR(
485 "No appropriate crypto function for given parameters");
489 /* Set up cipher session prerequisites */
490 if (cipher_set_prerequisites(sess, cipher_xform) != 0)
493 /* Set up authentication session prerequisites */
494 if (auth_set_prerequisites(sess, auth_xform) != 0)
500 /** Parse crypto xform chain and set private session parameters */
502 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess,
503 const struct rte_crypto_sym_xform *xform)
505 const struct rte_crypto_sym_xform *cipher_xform = NULL;
506 const struct rte_crypto_sym_xform *auth_xform = NULL;
510 /* Filter out spurious/broken requests */
514 sess->chain_order = armv8_crypto_get_chain_order(xform);
515 switch (sess->chain_order) {
516 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
517 cipher_xform = xform;
518 auth_xform = xform->next;
519 is_chained_op = true;
521 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
523 cipher_xform = xform->next;
524 is_chained_op = true;
527 is_chained_op = false;
532 sess->cipher.iv.offset = cipher_xform->cipher.iv.offset;
535 ret = armv8_crypto_set_session_chained_parameters(sess,
536 cipher_xform, auth_xform);
537 if (unlikely(ret != 0)) {
538 ARMV8_CRYPTO_LOG_ERR(
539 "Invalid/unsupported chained (cipher/auth) parameters");
543 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation");
550 /** Provide session for operation */
551 static inline struct armv8_crypto_session *
552 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op)
554 struct armv8_crypto_session *sess = NULL;
556 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
557 /* get existing session */
558 if (likely(op->sym->session != NULL)) {
559 sess = (struct armv8_crypto_session *)
560 get_session_private_data(
562 cryptodev_driver_id);
565 /* provide internal session */
567 void *_sess_private_data = NULL;
569 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
572 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
575 sess = (struct armv8_crypto_session *)_sess_private_data;
577 if (unlikely(armv8_crypto_set_session_parameters(sess,
578 op->sym->xform) != 0)) {
579 rte_mempool_put(qp->sess_mp, _sess);
580 rte_mempool_put(qp->sess_mp, _sess_private_data);
583 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
584 set_session_private_data(op->sym->session, cryptodev_driver_id,
588 if (unlikely(sess == NULL))
589 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
595 *------------------------------------------------------------------------------
597 *------------------------------------------------------------------------------
600 /*----------------------------------------------------------------------------*/
602 /** Process cipher operation */
604 process_armv8_chained_op
605 (struct rte_crypto_op *op, struct armv8_crypto_session *sess,
606 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
608 crypto_func_t crypto_func;
610 struct rte_mbuf *m_asrc, *m_adst;
611 uint8_t *csrc, *cdst;
612 uint8_t *adst, *asrc;
616 clen = op->sym->cipher.data.length;
617 alen = op->sym->auth.data.length;
619 csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
620 op->sym->cipher.data.offset);
621 cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
622 op->sym->cipher.data.offset);
624 switch (sess->chain_order) {
625 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
626 m_asrc = m_adst = mbuf_dst;
628 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
633 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
636 asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *,
637 op->sym->auth.data.offset);
639 switch (sess->auth.mode) {
640 case ARMV8_CRYPTO_AUTH_AS_AUTH:
641 /* Nothing to do here, just verify correct option */
643 case ARMV8_CRYPTO_AUTH_AS_HMAC:
644 arg.digest.hmac.key = sess->auth.hmac.key;
645 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad;
646 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad;
649 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
653 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) {
654 adst = op->sym->auth.digest.data;
656 adst = rte_pktmbuf_mtod_offset(m_adst,
658 op->sym->auth.data.offset +
659 op->sym->auth.data.length);
662 adst = (uint8_t *)rte_pktmbuf_append(m_asrc,
663 sess->auth.digest_length);
666 arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *,
667 sess->cipher.iv.offset);
668 arg.cipher.key = sess->cipher.key.data;
669 /* Acquire combined mode function */
670 crypto_func = sess->crypto_func;
671 ARMV8_CRYPTO_ASSERT(crypto_func != NULL);
672 error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg);
674 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
678 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
679 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
680 if (memcmp(adst, op->sym->auth.digest.data,
681 sess->auth.digest_length) != 0) {
682 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
684 /* Trim area used for digest from mbuf. */
685 rte_pktmbuf_trim(m_asrc,
686 sess->auth.digest_length);
690 /** Process crypto operation for mbuf */
692 process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op,
693 struct armv8_crypto_session *sess)
695 struct rte_mbuf *msrc, *mdst;
697 msrc = op->sym->m_src;
698 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
700 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
702 switch (sess->chain_order) {
703 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
704 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */
705 process_armv8_chained_op(op, sess, msrc, mdst);
708 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
712 /* Free session if a session-less crypto op */
713 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
714 memset(sess, 0, sizeof(struct armv8_crypto_session));
715 memset(op->sym->session, 0,
716 rte_cryptodev_get_header_session_size());
717 rte_mempool_put(qp->sess_mp, sess);
718 rte_mempool_put(qp->sess_mp, op->sym->session);
719 op->sym->session = NULL;
722 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
723 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
725 if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR))
732 *------------------------------------------------------------------------------
734 *------------------------------------------------------------------------------
739 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
742 struct armv8_crypto_session *sess;
743 struct armv8_crypto_qp *qp = queue_pair;
746 for (i = 0; i < nb_ops; i++) {
747 sess = get_session(qp, ops[i]);
748 if (unlikely(sess == NULL))
751 retval = process_op(qp, ops[i], sess);
752 if (unlikely(retval < 0))
756 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
758 qp->stats.enqueued_count += retval;
763 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
766 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
768 qp->stats.enqueue_err_count++;
774 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
777 struct armv8_crypto_qp *qp = queue_pair;
779 unsigned int nb_dequeued = 0;
781 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
782 (void **)ops, nb_ops, NULL);
783 qp->stats.dequeued_count += nb_dequeued;
788 /** Create ARMv8 crypto device */
790 cryptodev_armv8_crypto_create(const char *name,
791 struct rte_vdev_device *vdev,
792 struct rte_crypto_vdev_init_params *init_params)
794 struct rte_cryptodev *dev;
795 struct armv8_crypto_private *internals;
797 /* Check CPU for support for AES instruction set */
798 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
799 ARMV8_CRYPTO_LOG_ERR(
800 "AES instructions not supported by CPU");
804 /* Check CPU for support for SHA instruction set */
805 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) ||
806 !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) {
807 ARMV8_CRYPTO_LOG_ERR(
808 "SHA1/SHA2 instructions not supported by CPU");
812 /* Check CPU for support for Advance SIMD instruction set */
813 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
814 ARMV8_CRYPTO_LOG_ERR(
815 "Advanced SIMD instructions not supported by CPU");
819 if (init_params->name[0] == '\0')
820 snprintf(init_params->name, sizeof(init_params->name),
823 dev = rte_cryptodev_vdev_pmd_init(init_params->name,
824 sizeof(struct armv8_crypto_private),
825 init_params->socket_id,
828 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
832 dev->driver_id = cryptodev_driver_id;
833 dev->dev_ops = rte_armv8_crypto_pmd_ops;
835 /* register rx/tx burst functions for data path */
836 dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst;
837 dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst;
839 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
840 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
841 RTE_CRYPTODEV_FF_CPU_NEON |
842 RTE_CRYPTODEV_FF_CPU_ARM_CE;
844 /* Set vector instructions mode supported */
845 internals = dev->data->dev_private;
847 internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
848 internals->max_nb_sessions = init_params->max_nb_sessions;
853 ARMV8_CRYPTO_LOG_ERR(
854 "driver %s: cryptodev_armv8_crypto_create failed",
857 cryptodev_armv8_crypto_uninit(vdev);
861 /** Initialise ARMv8 crypto device */
863 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev)
865 struct rte_crypto_vdev_init_params init_params = {
866 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
867 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
872 const char *input_args;
874 name = rte_vdev_device_name(vdev);
877 input_args = rte_vdev_device_args(vdev);
878 rte_cryptodev_vdev_parse_init_params(&init_params, input_args);
880 RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
881 init_params.socket_id);
882 if (init_params.name[0] != '\0') {
883 RTE_LOG(INFO, PMD, " User defined name = %s\n",
886 RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n",
887 init_params.max_nb_queue_pairs);
888 RTE_LOG(INFO, PMD, " Max number of sessions = %d\n",
889 init_params.max_nb_sessions);
891 return cryptodev_armv8_crypto_create(name, vdev, &init_params);
894 /** Uninitialise ARMv8 crypto device */
896 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev)
900 name = rte_vdev_device_name(vdev);
905 "Closing ARMv8 crypto device %s on numa socket %u\n",
906 name, rte_socket_id());
911 static struct rte_vdev_driver armv8_crypto_drv = {
912 .probe = cryptodev_armv8_crypto_init,
913 .remove = cryptodev_armv8_crypto_uninit
916 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_drv);
917 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd);
918 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD,
919 "max_nb_queue_pairs=<int> "
920 "max_nb_sessions=<int> "
922 RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, cryptodev_driver_id);