X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fzuc%2Frte_zuc_pmd.c;h=42b669f18857b858a9507528a3b6bd913a0e04bb;hb=bbc42f34116c4baf04ead3e29e8d249311f31991;hp=313f4590bf0508ebaa267fb37f92429ca19d680e;hpb=f8e9989606e7a9548a25ac10e2daae4c8af230cb;p=dpdk.git diff --git a/drivers/crypto/zuc/rte_zuc_pmd.c b/drivers/crypto/zuc/rte_zuc_pmd.c index 313f4590bf..42b669f188 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd.c +++ b/drivers/crypto/zuc/rte_zuc_pmd.c @@ -10,8 +10,8 @@ #include #include -#include "rte_zuc_pmd_private.h" -#define ZUC_MAX_BURST 4 +#include "zuc_pmd_private.h" +#define ZUC_MAX_BURST 16 #define BYTE_LEN 8 static uint8_t cryptodev_driver_id; @@ -144,7 +144,8 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + if (rte_mempool_get(qp->sess_mp_priv, + (void **)&_sess_private_data)) return NULL; sess = (struct zuc_session *)_sess_private_data; @@ -152,7 +153,7 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) if (unlikely(zuc_set_session_parameters(sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); - rte_mempool_put(qp->sess_mp, _sess_private_data); + rte_mempool_put(qp->sess_mp_priv, _sess_private_data); sess = NULL; } op->sym->session = (struct rte_cryptodev_sym_session *)_sess; @@ -169,16 +170,17 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) /** Encrypt/decrypt mbufs. */ static uint8_t -process_zuc_cipher_op(struct rte_crypto_op **ops, +process_zuc_cipher_op(struct zuc_qp *qp, struct rte_crypto_op **ops, struct zuc_session **sessions, uint8_t num_ops) { unsigned i; uint8_t processed_ops = 0; - uint8_t *src[ZUC_MAX_BURST], *dst[ZUC_MAX_BURST]; - uint8_t *iv[ZUC_MAX_BURST]; + const void *src[ZUC_MAX_BURST]; + void *dst[ZUC_MAX_BURST]; + const void *iv[ZUC_MAX_BURST]; uint32_t num_bytes[ZUC_MAX_BURST]; - uint8_t *cipher_keys[ZUC_MAX_BURST]; + const void *cipher_keys[ZUC_MAX_BURST]; struct zuc_session *sess; for (i = 0; i < num_ops; i++) { @@ -221,7 +223,8 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, processed_ops++; } - sso_zuc_eea3_n_buffer(cipher_keys, iv, src, dst, + IMB_ZUC_EEA3_N_BUFFER(qp->mb_mgr, (const void **)cipher_keys, + (const void **)iv, (const void **)src, (void **)dst, num_bytes, processed_ops); return processed_ops; @@ -233,12 +236,13 @@ process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, struct zuc_session **sessions, uint8_t num_ops) { - unsigned i; + unsigned int i; uint8_t processed_ops = 0; - uint8_t *src; - uint32_t *dst; - uint32_t length_in_bits; - uint8_t *iv; + uint8_t *src[ZUC_MAX_BURST] = { 0 }; + uint32_t *dst[ZUC_MAX_BURST]; + uint32_t length_in_bits[ZUC_MAX_BURST] = { 0 }; + uint8_t *iv[ZUC_MAX_BURST] = { 0 }; + const void *hash_keys[ZUC_MAX_BURST] = { 0 }; struct zuc_session *sess; for (i = 0; i < num_ops; i++) { @@ -251,33 +255,42 @@ process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, sess = sessions[i]; - length_in_bits = ops[i]->sym->auth.data.length; + length_in_bits[i] = ops[i]->sym->auth.data.length; - src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + + src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); - iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, sess->auth_iv_offset); - if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { - dst = (uint32_t *)qp->temp_digest; - - sso_zuc_eia3_1_buffer(sess->pKey_hash, - iv, src, - length_in_bits, dst); - /* Verify digest. */ - if (memcmp(dst, ops[i]->sym->auth.digest.data, - ZUC_DIGEST_LENGTH) != 0) - ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - } else { - dst = (uint32_t *)ops[i]->sym->auth.digest.data; + hash_keys[i] = sess->pKey_hash; + if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) + dst[i] = (uint32_t *)qp->temp_digest; + else + dst[i] = (uint32_t *)ops[i]->sym->auth.digest.data; - sso_zuc_eia3_1_buffer(sess->pKey_hash, - iv, src, - length_in_bits, dst); - } +#if IMB_VERSION_NUM < IMB_VERSION(0, 53, 3) + IMB_ZUC_EIA3_1_BUFFER(qp->mb_mgr, hash_keys[i], + iv[i], src[i], length_in_bits[i], dst[i]); +#endif processed_ops++; } +#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3) + IMB_ZUC_EIA3_N_BUFFER(qp->mb_mgr, (const void **)hash_keys, + (const void * const *)iv, (const void * const *)src, + length_in_bits, dst, processed_ops); +#endif + + /* + * If tag needs to be verified, compare generated tag + * with attached tag + */ + for (i = 0; i < processed_ops; i++) + if (sessions[i]->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) + if (memcmp(dst[i], ops[i]->sym->auth.digest.data, + ZUC_DIGEST_LENGTH) != 0) + ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + return processed_ops; } @@ -293,7 +306,7 @@ process_ops(struct rte_crypto_op **ops, enum zuc_operation op_type, switch (op_type) { case ZUC_OP_ONLY_CIPHER: - processed_ops = process_zuc_cipher_op(ops, + processed_ops = process_zuc_cipher_op(qp, ops, sessions, num_ops); break; case ZUC_OP_ONLY_AUTH: @@ -301,14 +314,14 @@ process_ops(struct rte_crypto_op **ops, enum zuc_operation op_type, num_ops); break; case ZUC_OP_CIPHER_AUTH: - processed_ops = process_zuc_cipher_op(ops, sessions, + processed_ops = process_zuc_cipher_op(qp, ops, sessions, num_ops); process_zuc_hash_op(qp, ops, sessions, processed_ops); break; case ZUC_OP_AUTH_CIPHER: processed_ops = process_zuc_hash_op(qp, ops, sessions, num_ops); - process_zuc_cipher_op(ops, sessions, processed_ops); + process_zuc_cipher_op(qp, ops, sessions, processed_ops); break; default: /* Operation not supported. */ @@ -326,8 +339,9 @@ process_ops(struct rte_crypto_op **ops, enum zuc_operation op_type, if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { memset(sessions[i], 0, sizeof(struct zuc_session)); memset(ops[i]->sym->session, 0, - rte_cryptodev_sym_get_header_session_size()); - rte_mempool_put(qp->sess_mp, sessions[i]); + rte_cryptodev_sym_get_existing_header_session_size( + ops[i]->sym->session)); + rte_mempool_put(qp->sess_mp_priv, sessions[i]); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -455,8 +469,7 @@ cryptodev_zuc_create(const char *name, { struct rte_cryptodev *dev; struct zuc_private *internals; - uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; - + MB_MGR *mb_mgr; dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { @@ -464,6 +477,30 @@ cryptodev_zuc_create(const char *name, goto init_error; } + dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA | + RTE_CRYPTODEV_FF_SYM_SESSIONLESS | + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + + mb_mgr = alloc_mb_mgr(0); + if (mb_mgr == NULL) + return -ENOMEM; + + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512; + init_mb_mgr_avx512(mb_mgr); + } else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx2(mb_mgr); + } else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + init_mb_mgr_avx(mb_mgr); + } else { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + init_mb_mgr_sse(mb_mgr); + } + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_zuc_pmd_ops; @@ -471,11 +508,8 @@ cryptodev_zuc_create(const char *name, dev->dequeue_burst = zuc_pmd_dequeue_burst; dev->enqueue_burst = zuc_pmd_enqueue_burst; - dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | - RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | - cpu_flags; - internals = dev->data->dev_private; + internals->mb_mgr = mb_mgr; internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; @@ -516,6 +550,7 @@ cryptodev_zuc_remove(struct rte_vdev_device *vdev) struct rte_cryptodev *cryptodev; const char *name; + struct zuc_private *internals; name = rte_vdev_device_name(vdev); if (name == NULL) @@ -525,6 +560,10 @@ cryptodev_zuc_remove(struct rte_vdev_device *vdev) if (cryptodev == NULL) return -ENODEV; + internals = cryptodev->data->dev_private; + + free_mb_mgr(internals->mb_mgr); + return rte_cryptodev_pmd_destroy(cryptodev); } @@ -541,8 +580,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD, "socket_id="); RTE_PMD_REGISTER_CRYPTO_DRIVER(zuc_crypto_drv, cryptodev_zuc_pmd_drv.driver, cryptodev_driver_id); - -RTE_INIT(zuc_init_log) -{ - zuc_logtype_driver = rte_log_register("pmd.crypto.zuc"); -} +RTE_LOG_REGISTER_DEFAULT(zuc_logtype_driver, INFO);