4 * Copyright (C) Cavium, Inc. 2017.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Cavium, Inc nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #include <rte_common.h>
36 #include <rte_hexdump.h>
37 #include <rte_cryptodev.h>
38 #include <rte_cryptodev_pmd.h>
40 #include <rte_malloc.h>
41 #include <rte_cpuflags.h>
43 #include "armv8_crypto_defs.h"
45 #include "rte_armv8_pmd_private.h"
47 static uint8_t cryptodev_driver_id;
49 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev);
52 * Pointers to the supported combined mode crypto functions are stored
53 * in the static tables. Each combined (chained) cryptographic operation
54 * can be described by a set of numbers:
55 * - order: order of operations (cipher, auth) or (auth, cipher)
56 * - direction: encryption or decryption
57 * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc.
58 * - aalg: authentication algorithm such as SHA1, SHA256, etc.
59 * - keyl: cipher key length, for example 128, 192, 256 bits
61 * In order to quickly acquire each function pointer based on those numbers,
62 * a hierarchy of arrays is maintained. The final level, 3D array is indexed
63 * by the combined mode function parameters only (cipher algorithm,
64 * authentication algorithm and key length).
66 * This gives 3 memory accesses to obtain a function pointer instead of
67 * traversing the array manually and comparing function parameters on each loop.
81 * 3D array type for ARM Combined Mode crypto functions pointers.
82 * CRYPTO_CIPHER_MAX: max cipher ID number
83 * CRYPTO_AUTH_MAX: max auth ID number
84 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number
86 typedef const crypto_func_t
87 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
89 /* Evaluate to key length definition */
90 #define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl)
92 /* Local aliases for supported ciphers */
93 #define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC
94 /* Local aliases for supported hashes */
95 #define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC
96 #define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC
99 * Arrays containing pointers to particular cryptographic,
100 * combined mode functions.
101 * crypto_op_ca_encrypt: cipher (encrypt), authenticate
102 * crypto_op_ca_decrypt: cipher (decrypt), authenticate
103 * crypto_op_ac_encrypt: authenticate, cipher (encrypt)
104 * crypto_op_ac_decrypt: authenticate, cipher (decrypt)
106 static const crypto_func_tbl_t
107 crypto_op_ca_encrypt = {
108 /* [cipher alg][auth alg][key length] = crypto_function, */
109 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac,
110 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac,
113 static const crypto_func_tbl_t
114 crypto_op_ca_decrypt = {
118 static const crypto_func_tbl_t
119 crypto_op_ac_encrypt = {
123 static const crypto_func_tbl_t
124 crypto_op_ac_decrypt = {
125 /* [cipher alg][auth alg][key length] = crypto_function, */
126 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec,
127 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec,
131 * Arrays containing pointers to particular cryptographic function sets,
132 * covering given cipher operation directions (encrypt, decrypt)
133 * for each order of cipher and authentication pairs.
135 static const crypto_func_tbl_t *
136 crypto_cipher_auth[] = {
137 &crypto_op_ca_encrypt,
138 &crypto_op_ca_decrypt,
142 static const crypto_func_tbl_t *
143 crypto_auth_cipher[] = {
144 &crypto_op_ac_encrypt,
145 &crypto_op_ac_decrypt,
150 * Top level array containing pointers to particular cryptographic
151 * function sets, covering given order of chained operations.
152 * crypto_cipher_auth: cipher first, authenticate after
153 * crypto_auth_cipher: authenticate first, cipher after
155 static const crypto_func_tbl_t **
156 crypto_chain_order[] = {
163 * Extract particular combined mode crypto function from the 3D array.
165 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \
167 crypto_func_tbl_t *func_tbl = \
168 (crypto_chain_order[(order)])[(cop)]; \
170 ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \
173 /*----------------------------------------------------------------------------*/
176 * 2D array type for ARM key schedule functions pointers.
177 * CRYPTO_CIPHER_MAX: max cipher ID number
178 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number
180 typedef const crypto_key_sched_t
181 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
183 static const crypto_key_sched_tbl_t
184 crypto_key_sched_encrypt = {
185 /* [cipher alg][key length] = key_expand_func, */
186 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc,
189 static const crypto_key_sched_tbl_t
190 crypto_key_sched_decrypt = {
191 /* [cipher alg][key length] = key_expand_func, */
192 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec,
196 * Top level array containing pointers to particular key generation
197 * function sets, covering given operation direction.
198 * crypto_key_sched_encrypt: keys for encryption
199 * crypto_key_sched_decrypt: keys for decryption
201 static const crypto_key_sched_tbl_t *
202 crypto_key_sched_dir[] = {
203 &crypto_key_sched_encrypt,
204 &crypto_key_sched_decrypt,
209 * Extract particular combined mode crypto function from the 3D array.
211 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \
213 crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \
215 ((*ks_tbl)[(calg)][KEYL(keyl)]); \
218 /*----------------------------------------------------------------------------*/
221 *------------------------------------------------------------------------------
223 *------------------------------------------------------------------------------
226 /** Get xform chain order */
227 static enum armv8_crypto_chain_order
228 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform)
232 * This driver currently covers only chained operations.
233 * Ignore only cipher or only authentication operations
234 * or chains longer than 2 xform structures.
236 if (xform->next == NULL || xform->next->next != NULL)
237 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
239 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
240 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
241 return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER;
244 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
245 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
246 return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH;
249 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
253 auth_hmac_pad_prepare(struct armv8_crypto_session *sess,
254 const struct rte_crypto_sym_xform *xform)
258 /* Generate i_key_pad and o_key_pad */
259 memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad));
260 rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key,
261 xform->auth.key.length);
262 memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad));
263 rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key,
264 xform->auth.key.length);
266 * XOR key with IPAD/OPAD values to obtain i_key_pad
268 * Byte-by-byte operation may seem to be the less efficient
269 * here but in fact it's the opposite.
270 * The result ASM code is likely operate on NEON registers
271 * (load auth key to Qx, load IPAD/OPAD to multiple
272 * elements of Qy, eor 128 bits at once).
274 for (i = 0; i < SHA_BLOCK_MAX; i++) {
275 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE;
276 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE;
281 auth_set_prerequisites(struct armv8_crypto_session *sess,
282 const struct rte_crypto_sym_xform *xform)
284 uint8_t partial[64] = { 0 };
287 switch (xform->auth.algo) {
288 case RTE_CRYPTO_AUTH_SHA1_HMAC:
290 * Generate authentication key, i_key_pad and o_key_pad.
292 /* Zero memory under key */
293 memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE);
296 * Now copy the given authentication key to the session
299 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
300 xform->auth.key.length);
302 /* Prepare HMAC padding: key|pattern */
303 auth_hmac_pad_prepare(sess, xform);
305 * Calculate partial hash values for i_key_pad and o_key_pad.
306 * Will be used as initialization state for final HMAC.
308 error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad,
309 partial, SHA1_BLOCK_SIZE);
312 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE);
314 error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad,
315 partial, SHA1_BLOCK_SIZE);
318 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE);
321 case RTE_CRYPTO_AUTH_SHA256_HMAC:
323 * Generate authentication key, i_key_pad and o_key_pad.
325 /* Zero memory under key */
326 memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE);
329 * Now copy the given authentication key to the session
332 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
333 xform->auth.key.length);
335 /* Prepare HMAC padding: key|pattern */
336 auth_hmac_pad_prepare(sess, xform);
338 * Calculate partial hash values for i_key_pad and o_key_pad.
339 * Will be used as initialization state for final HMAC.
341 error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad,
342 partial, SHA256_BLOCK_SIZE);
345 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE);
347 error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad,
348 partial, SHA256_BLOCK_SIZE);
351 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE);
362 cipher_set_prerequisites(struct armv8_crypto_session *sess,
363 const struct rte_crypto_sym_xform *xform)
365 crypto_key_sched_t cipher_key_sched;
367 cipher_key_sched = sess->cipher.key_sched;
368 if (likely(cipher_key_sched != NULL)) {
369 /* Set up cipher session key */
370 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data);
377 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess,
378 const struct rte_crypto_sym_xform *cipher_xform,
379 const struct rte_crypto_sym_xform *auth_xform)
381 enum armv8_crypto_chain_order order;
382 enum armv8_crypto_cipher_operation cop;
383 enum rte_crypto_cipher_algorithm calg;
384 enum rte_crypto_auth_algorithm aalg;
386 /* Validate and prepare scratch order of combined operations */
387 switch (sess->chain_order) {
388 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
389 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
390 order = sess->chain_order;
395 /* Select cipher direction */
396 sess->cipher.direction = cipher_xform->cipher.op;
397 /* Select cipher key */
398 sess->cipher.key.length = cipher_xform->cipher.key.length;
399 /* Set cipher direction */
400 cop = sess->cipher.direction;
401 /* Set cipher algorithm */
402 calg = cipher_xform->cipher.algo;
404 /* Select cipher algo */
406 /* Cover supported cipher algorithms */
407 case RTE_CRYPTO_CIPHER_AES_CBC:
408 sess->cipher.algo = calg;
409 /* IV len is always 16 bytes (block size) for AES CBC */
410 sess->cipher.iv.length = 16;
415 /* Select auth generate/verify */
416 sess->auth.operation = auth_xform->auth.op;
418 /* Select auth algo */
419 switch (auth_xform->auth.algo) {
420 /* Cover supported hash algorithms */
421 case RTE_CRYPTO_AUTH_SHA1_HMAC:
422 case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */
423 aalg = auth_xform->auth.algo;
424 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC;
430 /* Set the digest length */
431 sess->auth.digest_length = auth_xform->auth.digest_length;
433 /* Verify supported key lengths and extract proper algorithm */
434 switch (cipher_xform->cipher.key.length << 3) {
437 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128);
438 sess->cipher.key_sched =
439 CRYPTO_GET_KEY_SCHED(cop, calg, 128);
443 /* These key lengths are not supported yet */
444 default: /* Fall through */
445 sess->crypto_func = NULL;
446 sess->cipher.key_sched = NULL;
450 if (unlikely(sess->crypto_func == NULL)) {
452 * If we got here that means that there must be a bug
453 * in the algorithms selection above. Nevertheless keep
454 * it here to catch bug immediately and avoid NULL pointer
455 * dereference in OPs processing.
457 ARMV8_CRYPTO_LOG_ERR(
458 "No appropriate crypto function for given parameters");
462 /* Set up cipher session prerequisites */
463 if (cipher_set_prerequisites(sess, cipher_xform) != 0)
466 /* Set up authentication session prerequisites */
467 if (auth_set_prerequisites(sess, auth_xform) != 0)
473 /** Parse crypto xform chain and set private session parameters */
475 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess,
476 const struct rte_crypto_sym_xform *xform)
478 const struct rte_crypto_sym_xform *cipher_xform = NULL;
479 const struct rte_crypto_sym_xform *auth_xform = NULL;
483 /* Filter out spurious/broken requests */
487 sess->chain_order = armv8_crypto_get_chain_order(xform);
488 switch (sess->chain_order) {
489 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
490 cipher_xform = xform;
491 auth_xform = xform->next;
492 is_chained_op = true;
494 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
496 cipher_xform = xform->next;
497 is_chained_op = true;
500 is_chained_op = false;
505 sess->cipher.iv.offset = cipher_xform->cipher.iv.offset;
508 ret = armv8_crypto_set_session_chained_parameters(sess,
509 cipher_xform, auth_xform);
510 if (unlikely(ret != 0)) {
511 ARMV8_CRYPTO_LOG_ERR(
512 "Invalid/unsupported chained (cipher/auth) parameters");
516 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation");
523 /** Provide session for operation */
524 static inline struct armv8_crypto_session *
525 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op)
527 struct armv8_crypto_session *sess = NULL;
529 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
530 /* get existing session */
531 if (likely(op->sym->session != NULL)) {
532 sess = (struct armv8_crypto_session *)
533 get_session_private_data(
535 cryptodev_driver_id);
538 /* provide internal session */
540 void *_sess_private_data = NULL;
542 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
545 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
548 sess = (struct armv8_crypto_session *)_sess_private_data;
550 if (unlikely(armv8_crypto_set_session_parameters(sess,
551 op->sym->xform) != 0)) {
552 rte_mempool_put(qp->sess_mp, _sess);
553 rte_mempool_put(qp->sess_mp, _sess_private_data);
556 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
557 set_session_private_data(op->sym->session, cryptodev_driver_id,
561 if (unlikely(sess == NULL))
562 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
568 *------------------------------------------------------------------------------
570 *------------------------------------------------------------------------------
573 /*----------------------------------------------------------------------------*/
575 /** Process cipher operation */
577 process_armv8_chained_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op,
578 struct armv8_crypto_session *sess,
579 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
581 crypto_func_t crypto_func;
583 struct rte_mbuf *m_asrc, *m_adst;
584 uint8_t *csrc, *cdst;
585 uint8_t *adst, *asrc;
589 clen = op->sym->cipher.data.length;
590 alen = op->sym->auth.data.length;
592 csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
593 op->sym->cipher.data.offset);
594 cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
595 op->sym->cipher.data.offset);
597 switch (sess->chain_order) {
598 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
599 m_asrc = m_adst = mbuf_dst;
601 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
606 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
609 asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *,
610 op->sym->auth.data.offset);
612 switch (sess->auth.mode) {
613 case ARMV8_CRYPTO_AUTH_AS_AUTH:
614 /* Nothing to do here, just verify correct option */
616 case ARMV8_CRYPTO_AUTH_AS_HMAC:
617 arg.digest.hmac.key = sess->auth.hmac.key;
618 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad;
619 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad;
622 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
626 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) {
627 adst = op->sym->auth.digest.data;
629 adst = rte_pktmbuf_mtod_offset(m_adst,
631 op->sym->auth.data.offset +
632 op->sym->auth.data.length);
635 adst = qp->temp_digest;
638 arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *,
639 sess->cipher.iv.offset);
640 arg.cipher.key = sess->cipher.key.data;
641 /* Acquire combined mode function */
642 crypto_func = sess->crypto_func;
643 ARMV8_CRYPTO_ASSERT(crypto_func != NULL);
644 error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg);
646 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
650 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
651 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
652 if (memcmp(adst, op->sym->auth.digest.data,
653 sess->auth.digest_length) != 0) {
654 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
659 /** Process crypto operation for mbuf */
661 process_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op,
662 struct armv8_crypto_session *sess)
664 struct rte_mbuf *msrc, *mdst;
666 msrc = op->sym->m_src;
667 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
669 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
671 switch (sess->chain_order) {
672 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
673 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */
674 process_armv8_chained_op(qp, op, sess, msrc, mdst);
677 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
681 /* Free session if a session-less crypto op */
682 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
683 memset(sess, 0, sizeof(struct armv8_crypto_session));
684 memset(op->sym->session, 0,
685 rte_cryptodev_get_header_session_size());
686 rte_mempool_put(qp->sess_mp, sess);
687 rte_mempool_put(qp->sess_mp, op->sym->session);
688 op->sym->session = NULL;
691 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
692 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
694 if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR))
701 *------------------------------------------------------------------------------
703 *------------------------------------------------------------------------------
708 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
711 struct armv8_crypto_session *sess;
712 struct armv8_crypto_qp *qp = queue_pair;
715 for (i = 0; i < nb_ops; i++) {
716 sess = get_session(qp, ops[i]);
717 if (unlikely(sess == NULL))
720 retval = process_op(qp, ops[i], sess);
721 if (unlikely(retval < 0))
725 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
727 qp->stats.enqueued_count += retval;
732 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
735 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
737 qp->stats.enqueue_err_count++;
743 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
746 struct armv8_crypto_qp *qp = queue_pair;
748 unsigned int nb_dequeued = 0;
750 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
751 (void **)ops, nb_ops, NULL);
752 qp->stats.dequeued_count += nb_dequeued;
757 /** Create ARMv8 crypto device */
759 cryptodev_armv8_crypto_create(const char *name,
760 struct rte_vdev_device *vdev,
761 struct rte_cryptodev_pmd_init_params *init_params)
763 struct rte_cryptodev *dev;
764 struct armv8_crypto_private *internals;
766 /* Check CPU for support for AES instruction set */
767 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
768 ARMV8_CRYPTO_LOG_ERR(
769 "AES instructions not supported by CPU");
773 /* Check CPU for support for SHA instruction set */
774 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) ||
775 !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) {
776 ARMV8_CRYPTO_LOG_ERR(
777 "SHA1/SHA2 instructions not supported by CPU");
781 /* Check CPU for support for Advance SIMD instruction set */
782 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
783 ARMV8_CRYPTO_LOG_ERR(
784 "Advanced SIMD instructions not supported by CPU");
788 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
790 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
794 dev->driver_id = cryptodev_driver_id;
795 dev->dev_ops = rte_armv8_crypto_pmd_ops;
797 /* register rx/tx burst functions for data path */
798 dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst;
799 dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst;
801 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
802 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
803 RTE_CRYPTODEV_FF_CPU_NEON |
804 RTE_CRYPTODEV_FF_CPU_ARM_CE;
806 /* Set vector instructions mode supported */
807 internals = dev->data->dev_private;
809 internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
810 internals->max_nb_sessions = init_params->max_nb_sessions;
815 ARMV8_CRYPTO_LOG_ERR(
816 "driver %s: cryptodev_armv8_crypto_create failed",
819 cryptodev_armv8_crypto_uninit(vdev);
823 /** Initialise ARMv8 crypto device */
825 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev)
827 struct rte_cryptodev_pmd_init_params init_params = {
829 sizeof(struct armv8_crypto_private),
831 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
832 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
835 const char *input_args;
837 name = rte_vdev_device_name(vdev);
840 input_args = rte_vdev_device_args(vdev);
841 rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
843 return cryptodev_armv8_crypto_create(name, vdev, &init_params);
846 /** Uninitialise ARMv8 crypto device */
848 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev)
850 struct rte_cryptodev *cryptodev;
853 name = rte_vdev_device_name(vdev);
858 "Closing ARMv8 crypto device %s on numa socket %u\n",
859 name, rte_socket_id());
861 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
862 if (cryptodev == NULL)
865 return rte_cryptodev_pmd_destroy(cryptodev);
868 static struct rte_vdev_driver armv8_crypto_pmd_drv = {
869 .probe = cryptodev_armv8_crypto_init,
870 .remove = cryptodev_armv8_crypto_uninit
873 static struct cryptodev_driver armv8_crypto_drv;
875 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_pmd_drv);
876 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd);
877 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD,
878 "max_nb_queue_pairs=<int> "
879 "max_nb_sessions=<int> "
881 RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, armv8_crypto_pmd_drv,
882 cryptodev_driver_id);