X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=drivers%2Fcrypto%2Fzuc%2Frte_zuc_pmd.c;h=a9ff318281a34d1c3c573fb84b633540330ecdb5;hb=330a70b773f04060e3026004d6291182a5cbefaf;hp=cd38f30309e59fe9b004bcf6ae17cdc8f12dda68;hpb=17c36a6746419ae8d1a14b1d1345d4a892511c57;p=dpdk.git diff --git a/drivers/crypto/zuc/rte_zuc_pmd.c b/drivers/crypto/zuc/rte_zuc_pmd.c index cd38f30309..a9ff318281 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd.c +++ b/drivers/crypto/zuc/rte_zuc_pmd.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2016-2017 Intel Corporation + * Copyright(c) 2016-2018 Intel Corporation */ #include @@ -10,9 +10,8 @@ #include #include -#include "rte_zuc_pmd_private.h" - -#define ZUC_MAX_BURST 4 +#include "zuc_pmd_private.h" +#define ZUC_MAX_BURST 16 #define BYTE_LEN 8 static uint8_t cryptodev_driver_id; @@ -78,7 +77,7 @@ zuc_set_session_parameters(struct zuc_session *sess, break; case ZUC_OP_NOT_SUPPORTED: default: - ZUC_LOG_ERR("Unsupported operation chain order parameter"); + ZUC_LOG(ERR, "Unsupported operation chain order parameter"); return -ENOTSUP; } @@ -88,7 +87,7 @@ zuc_set_session_parameters(struct zuc_session *sess, return -ENOTSUP; if (cipher_xform->cipher.iv.length != ZUC_IV_KEY_LENGTH) { - ZUC_LOG_ERR("Wrong IV length"); + ZUC_LOG(ERR, "Wrong IV length"); return -EINVAL; } sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; @@ -104,14 +103,14 @@ zuc_set_session_parameters(struct zuc_session *sess, return -ENOTSUP; if (auth_xform->auth.digest_length != ZUC_DIGEST_LENGTH) { - ZUC_LOG_ERR("Wrong digest length"); + ZUC_LOG(ERR, "Wrong digest length"); return -EINVAL; } sess->auth_op = auth_xform->auth.op; if (auth_xform->auth.iv.length != ZUC_IV_KEY_LENGTH) { - ZUC_LOG_ERR("Wrong IV length"); + ZUC_LOG(ERR, "Wrong IV length"); return -EINVAL; } sess->auth_iv_offset = auth_xform->auth.iv.offset; @@ -135,7 +134,7 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { if (likely(op->sym->session != NULL)) - sess = (struct zuc_session *)get_session_private_data( + sess = (struct zuc_session *)get_sym_session_private_data( op->sym->session, cryptodev_driver_id); } else { @@ -145,7 +144,8 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + if (rte_mempool_get(qp->sess_mp_priv, + (void **)&_sess_private_data)) return NULL; sess = (struct zuc_session *)_sess_private_data; @@ -153,12 +153,12 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) if (unlikely(zuc_set_session_parameters(sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); - rte_mempool_put(qp->sess_mp, _sess_private_data); + rte_mempool_put(qp->sess_mp_priv, _sess_private_data); sess = NULL; } op->sym->session = (struct rte_cryptodev_sym_session *)_sess; - set_session_private_data(op->sym->session, cryptodev_driver_id, - _sess_private_data); + set_sym_session_private_data(op->sym->session, + cryptodev_driver_id, _sess_private_data); } if (unlikely(sess == NULL)) @@ -170,16 +170,17 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) /** Encrypt/decrypt mbufs. */ static uint8_t -process_zuc_cipher_op(struct rte_crypto_op **ops, +process_zuc_cipher_op(struct zuc_qp *qp, struct rte_crypto_op **ops, struct zuc_session **sessions, uint8_t num_ops) { unsigned i; uint8_t processed_ops = 0; - uint8_t *src[ZUC_MAX_BURST], *dst[ZUC_MAX_BURST]; - uint8_t *iv[ZUC_MAX_BURST]; + const void *src[ZUC_MAX_BURST]; + void *dst[ZUC_MAX_BURST]; + const void *iv[ZUC_MAX_BURST]; uint32_t num_bytes[ZUC_MAX_BURST]; - uint8_t *cipher_keys[ZUC_MAX_BURST]; + const void *cipher_keys[ZUC_MAX_BURST]; struct zuc_session *sess; for (i = 0; i < num_ops; i++) { @@ -187,7 +188,7 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, || ((ops[i]->sym->cipher.data.offset % BYTE_LEN) != 0)) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - ZUC_LOG_ERR("Data Length or offset"); + ZUC_LOG(ERR, "Data Length or offset"); break; } @@ -198,7 +199,7 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->m_dst != NULL && !rte_pktmbuf_is_contiguous( ops[i]->sym->m_dst))) { - ZUC_LOG_ERR("PMD supports only contiguous mbufs, " + ZUC_LOG(ERR, "PMD supports only contiguous mbufs, " "op (%p) provides noncontiguous mbuf as " "source/destination buffer.\n", ops[i]); ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -222,7 +223,8 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, processed_ops++; } - sso_zuc_eea3_n_buffer(cipher_keys, iv, src, dst, + IMB_ZUC_EEA3_N_BUFFER(qp->mb_mgr, (const void **)cipher_keys, + (const void **)iv, (const void **)src, (void **)dst, num_bytes, processed_ops); return processed_ops; @@ -234,51 +236,61 @@ process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, struct zuc_session **sessions, uint8_t num_ops) { - unsigned i; + unsigned int i; uint8_t processed_ops = 0; - uint8_t *src; - uint32_t *dst; - uint32_t length_in_bits; - uint8_t *iv; + uint8_t *src[ZUC_MAX_BURST]; + uint32_t *dst[ZUC_MAX_BURST]; + uint32_t length_in_bits[ZUC_MAX_BURST]; + uint8_t *iv[ZUC_MAX_BURST]; + const void *hash_keys[ZUC_MAX_BURST]; struct zuc_session *sess; for (i = 0; i < num_ops; i++) { /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - ZUC_LOG_ERR("Offset"); + ZUC_LOG(ERR, "Offset"); break; } sess = sessions[i]; - length_in_bits = ops[i]->sym->auth.data.length; + length_in_bits[i] = ops[i]->sym->auth.data.length; - src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + + src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); - iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, sess->auth_iv_offset); - if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { - dst = (uint32_t *)qp->temp_digest; - - sso_zuc_eia3_1_buffer(sess->pKey_hash, - iv, src, - length_in_bits, dst); - /* Verify digest. */ - if (memcmp(dst, ops[i]->sym->auth.digest.data, - ZUC_DIGEST_LENGTH) != 0) - ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - } else { - dst = (uint32_t *)ops[i]->sym->auth.digest.data; + hash_keys[i] = sess->pKey_hash; + if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) + dst[i] = (uint32_t *)qp->temp_digest; + else + dst[i] = (uint32_t *)ops[i]->sym->auth.digest.data; - sso_zuc_eia3_1_buffer(sess->pKey_hash, - iv, src, - length_in_bits, dst); - } +#if IMB_VERSION_NUM < IMB_VERSION(0, 53, 3) + IMB_ZUC_EIA3_1_BUFFER(qp->mb_mgr, hash_keys[i], + iv[i], src[i], length_in_bits[i], dst[i]); +#endif processed_ops++; } +#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3) + IMB_ZUC_EIA3_N_BUFFER(qp->mb_mgr, (const void **)hash_keys, + (const void * const *)iv, (const void * const *)src, + length_in_bits, dst, processed_ops); +#endif + + /* + * If tag needs to be verified, compare generated tag + * with attached tag + */ + for (i = 0; i < processed_ops; i++) + if (sessions[i]->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) + if (memcmp(dst[i], ops[i]->sym->auth.digest.data, + ZUC_DIGEST_LENGTH) != 0) + ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + return processed_ops; } @@ -294,7 +306,7 @@ process_ops(struct rte_crypto_op **ops, enum zuc_operation op_type, switch (op_type) { case ZUC_OP_ONLY_CIPHER: - processed_ops = process_zuc_cipher_op(ops, + processed_ops = process_zuc_cipher_op(qp, ops, sessions, num_ops); break; case ZUC_OP_ONLY_AUTH: @@ -302,14 +314,14 @@ process_ops(struct rte_crypto_op **ops, enum zuc_operation op_type, num_ops); break; case ZUC_OP_CIPHER_AUTH: - processed_ops = process_zuc_cipher_op(ops, sessions, + processed_ops = process_zuc_cipher_op(qp, ops, sessions, num_ops); process_zuc_hash_op(qp, ops, sessions, processed_ops); break; case ZUC_OP_AUTH_CIPHER: processed_ops = process_zuc_hash_op(qp, ops, sessions, num_ops); - process_zuc_cipher_op(ops, sessions, processed_ops); + process_zuc_cipher_op(qp, ops, sessions, processed_ops); break; default: /* Operation not supported. */ @@ -327,8 +339,9 @@ process_ops(struct rte_crypto_op **ops, enum zuc_operation op_type, if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { memset(sessions[i], 0, sizeof(struct zuc_session)); memset(ops[i]->sym->session, 0, - rte_cryptodev_get_header_session_size()); - rte_mempool_put(qp->sess_mp, sessions[i]); + rte_cryptodev_sym_get_existing_header_session_size( + ops[i]->sym->session)); + rte_mempool_put(qp->sess_mp_priv, sessions[i]); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -456,15 +469,38 @@ cryptodev_zuc_create(const char *name, { struct rte_cryptodev *dev; struct zuc_private *internals; - uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; - + MB_MGR *mb_mgr; dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { - ZUC_LOG_ERR("failed to create cryptodev vdev"); + ZUC_LOG(ERR, "failed to create cryptodev vdev"); goto init_error; } + dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA | + RTE_CRYPTODEV_FF_SYM_SESSIONLESS | + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + + mb_mgr = alloc_mb_mgr(0); + if (mb_mgr == NULL) + return -ENOMEM; + + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512; + init_mb_mgr_avx512(mb_mgr); + } else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx2(mb_mgr); + } else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + init_mb_mgr_avx(mb_mgr); + } else { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + init_mb_mgr_sse(mb_mgr); + } + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_zuc_pmd_ops; @@ -472,18 +508,14 @@ cryptodev_zuc_create(const char *name, dev->dequeue_burst = zuc_pmd_dequeue_burst; dev->enqueue_burst = zuc_pmd_enqueue_burst; - dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | - RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | - cpu_flags; - internals = dev->data->dev_private; + internals->mb_mgr = mb_mgr; internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; - internals->max_nb_sessions = init_params->max_nb_sessions; return 0; init_error: - ZUC_LOG_ERR("driver %s: cryptodev_zuc_create failed", + ZUC_LOG(ERR, "driver %s: failed", init_params->name); cryptodev_zuc_remove(vdev); @@ -497,8 +529,7 @@ cryptodev_zuc_probe(struct rte_vdev_device *vdev) "", sizeof(struct zuc_private), rte_socket_id(), - RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS }; const char *name; const char *input_args; @@ -519,6 +550,7 @@ cryptodev_zuc_remove(struct rte_vdev_device *vdev) struct rte_cryptodev *cryptodev; const char *name; + struct zuc_private *internals; name = rte_vdev_device_name(vdev); if (name == NULL) @@ -528,6 +560,10 @@ cryptodev_zuc_remove(struct rte_vdev_device *vdev) if (cryptodev == NULL) return -ENODEV; + internals = cryptodev->data->dev_private; + + free_mb_mgr(internals->mb_mgr); + return rte_cryptodev_pmd_destroy(cryptodev); } @@ -541,7 +577,7 @@ static struct cryptodev_driver zuc_crypto_drv; RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ZUC_PMD, cryptodev_zuc_pmd_drv); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD, "max_nb_queue_pairs= " - "max_nb_sessions= " "socket_id="); RTE_PMD_REGISTER_CRYPTO_DRIVER(zuc_crypto_drv, cryptodev_zuc_pmd_drv.driver, cryptodev_driver_id); +RTE_LOG_REGISTER(zuc_logtype_driver, pmd.crypto.zuc, INFO);