From 81fe96a0cece6083981cc49e81a94215e4ef02ff Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 10 Apr 2019 01:41:34 +0100 Subject: [PATCH] crypto/aesni_gcm: use architecture independent API IPSec Multi-buffer library has recently added API that sets pointers depending on the architecture, at initiliazation, in version 0.52. AESNI MB PMD was updated with these changes, but not the AESNI GCM PMD, which also uses the same library. This makes the PMD to be only compatible with version 0.52, but makes both PMD consistent on version compatibility, plus adds support for AVX512. Signed-off-by: Pablo de Lara --- doc/guides/cryptodevs/aesni_gcm.rst | 9 ++- doc/guides/cryptodevs/features/aesni_gcm.ini | 1 + drivers/crypto/aesni_gcm/aesni_gcm_ops.h | 66 +++------------- drivers/crypto/aesni_gcm/aesni_gcm_pmd.c | 77 ++++++++++++++++--- drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c | 4 +- .../crypto/aesni_gcm/aesni_gcm_pmd_private.h | 6 +- 6 files changed, 89 insertions(+), 74 deletions(-) diff --git a/doc/guides/cryptodevs/aesni_gcm.rst b/doc/guides/cryptodevs/aesni_gcm.rst index e0346080a9..9a8bc9323f 100644 --- a/doc/guides/cryptodevs/aesni_gcm.rst +++ b/doc/guides/cryptodevs/aesni_gcm.rst @@ -1,5 +1,5 @@ .. SPDX-License-Identifier: BSD-3-Clause - Copyright(c) 2016-2017 Intel Corporation. + Copyright(c) 2016-2019 Intel Corporation. AES-NI GCM Crypto Poll Mode Driver ================================== @@ -36,8 +36,8 @@ Installation To build DPDK with the AESNI_GCM_PMD the user is required to download the multi-buffer library from `here `_ and compile it on their user system before building DPDK. -The latest version of the library supported by this PMD is v0.50, which -can be downloaded in ``_. +The latest version of the library supported by this PMD is v0.52, which +can be downloaded in ``_. .. code-block:: console @@ -57,7 +57,8 @@ and the external crypto libraries supported by them: 16.04 - 16.11 Multi-buffer library 0.43 - 0.44 17.02 - 17.05 ISA-L Crypto v2.18 17.08 - 18.02 Multi-buffer library 0.46 - 0.48 - 18.05+ Multi-buffer library 0.49+ + 18.05 - 19.02 Multi-buffer library 0.49 - 0.52 + 19.05+ Multi-buffer library 0.52 ============= ================================ diff --git a/doc/guides/cryptodevs/features/aesni_gcm.ini b/doc/guides/cryptodevs/features/aesni_gcm.ini index b9e9c90605..e32e83c203 100644 --- a/doc/guides/cryptodevs/features/aesni_gcm.ini +++ b/doc/guides/cryptodevs/features/aesni_gcm.ini @@ -10,6 +10,7 @@ CPU AESNI = Y CPU SSE = Y CPU AVX = Y CPU AVX2 = Y +CPU AVX512 = Y OOP SGL In LB Out = Y OOP LB In LB Out = Y ; diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h index 4506166986..e272f10671 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h @@ -17,24 +17,24 @@ enum aesni_gcm_vector_mode { RTE_AESNI_GCM_SSE, RTE_AESNI_GCM_AVX, RTE_AESNI_GCM_AVX2, + RTE_AESNI_GCM_AVX512, RTE_AESNI_GCM_VECTOR_NUM }; enum aesni_gcm_key { - AESNI_GCM_KEY_128, - AESNI_GCM_KEY_192, - AESNI_GCM_KEY_256, - AESNI_GCM_KEY_NUM + GCM_KEY_128 = 0, + GCM_KEY_192, + GCM_KEY_256, + GCM_KEY_NUM }; - typedef void (*aesni_gcm_t)(const struct gcm_key_data *gcm_key_data, struct gcm_context_data *gcm_ctx_data, uint8_t *out, const uint8_t *in, uint64_t plaintext_len, const uint8_t *iv, const uint8_t *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); -typedef void (*aesni_gcm_precomp_t)(const void *key, struct gcm_key_data *gcm_data); +typedef void (*aesni_gcm_pre_t)(const void *key, struct gcm_key_data *gcm_data); typedef void (*aesni_gcm_init_t)(const struct gcm_key_data *gcm_key_data, struct gcm_context_data *gcm_ctx_data, @@ -57,60 +57,12 @@ typedef void (*aesni_gcm_finalize_t)(const struct gcm_key_data *gcm_key_data, struct aesni_gcm_ops { aesni_gcm_t enc; /**< GCM encode function pointer */ aesni_gcm_t dec; /**< GCM decode function pointer */ - aesni_gcm_precomp_t precomp; /**< GCM pre-compute */ + aesni_gcm_pre_t pre; /**< GCM pre-compute */ aesni_gcm_init_t init; aesni_gcm_update_t update_enc; aesni_gcm_update_t update_dec; - aesni_gcm_finalize_t finalize; + aesni_gcm_finalize_t finalize_enc; + aesni_gcm_finalize_t finalize_dec; }; -#define AES_GCM_FN(keylen, arch) \ -aes_gcm_enc_##keylen##_##arch,\ -aes_gcm_dec_##keylen##_##arch,\ -aes_gcm_pre_##keylen##_##arch,\ -aes_gcm_init_##keylen##_##arch,\ -aes_gcm_enc_##keylen##_update_##arch,\ -aes_gcm_dec_##keylen##_update_##arch,\ -aes_gcm_enc_##keylen##_finalize_##arch, - -static const struct aesni_gcm_ops gcm_ops[RTE_AESNI_GCM_VECTOR_NUM][AESNI_GCM_KEY_NUM] = { - [RTE_AESNI_GCM_NOT_SUPPORTED] = { - [AESNI_GCM_KEY_128] = {NULL}, - [AESNI_GCM_KEY_192] = {NULL}, - [AESNI_GCM_KEY_256] = {NULL} - }, - [RTE_AESNI_GCM_SSE] = { - [AESNI_GCM_KEY_128] = { - AES_GCM_FN(128, sse) - }, - [AESNI_GCM_KEY_192] = { - AES_GCM_FN(192, sse) - }, - [AESNI_GCM_KEY_256] = { - AES_GCM_FN(256, sse) - } - }, - [RTE_AESNI_GCM_AVX] = { - [AESNI_GCM_KEY_128] = { - AES_GCM_FN(128, avx_gen2) - }, - [AESNI_GCM_KEY_192] = { - AES_GCM_FN(192, avx_gen2) - }, - [AESNI_GCM_KEY_256] = { - AES_GCM_FN(256, avx_gen2) - } - }, - [RTE_AESNI_GCM_AVX2] = { - [AESNI_GCM_KEY_128] = { - AES_GCM_FN(128, avx_gen4) - }, - [AESNI_GCM_KEY_192] = { - AES_GCM_FN(192, avx_gen4) - }, - [AESNI_GCM_KEY_256] = { - AES_GCM_FN(256, avx_gen4) - } - } -}; #endif /* _AESNI_GCM_OPS_H_ */ diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index 948ff07634..49ee317086 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -89,20 +89,20 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, /* Check key length and calculate GCM pre-compute. */ switch (key_length) { case 16: - sess->key = AESNI_GCM_KEY_128; + sess->key = GCM_KEY_128; break; case 24: - sess->key = AESNI_GCM_KEY_192; + sess->key = GCM_KEY_192; break; case 32: - sess->key = AESNI_GCM_KEY_256; + sess->key = GCM_KEY_256; break; default: AESNI_GCM_LOG(ERR, "Invalid key length"); return -EINVAL; } - gcm_ops[sess->key].precomp(key, &sess->gdata_key); + gcm_ops[sess->key].pre(key, &sess->gdata_key); /* Digest check */ if (sess->req_digest_length > 16) { @@ -275,7 +275,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, else tag = sym_op->aead.digest.data; - qp->ops[session->key].finalize(&session->gdata_key, + qp->ops[session->key].finalize_enc(&session->gdata_key, &qp->gdata_ctx, tag, session->gen_digest_length); @@ -309,7 +309,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, } tag = qp->temp_digest; - qp->ops[session->key].finalize(&session->gdata_key, + qp->ops[session->key].finalize_dec(&session->gdata_key, &qp->gdata_ctx, tag, session->gen_digest_length); @@ -323,7 +323,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, tag = qp->temp_digest; else tag = sym_op->auth.digest.data; - qp->ops[session->key].finalize(&session->gdata_key, + qp->ops[session->key].finalize_enc(&session->gdata_key, &qp->gdata_ctx, tag, session->gen_digest_length); @@ -339,7 +339,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, * the bytes passed. */ tag = qp->temp_digest; - qp->ops[session->key].finalize(&session->gdata_key, + qp->ops[session->key].finalize_enc(&session->gdata_key, &qp->gdata_ctx, tag, session->gen_digest_length); @@ -489,6 +489,7 @@ aesni_gcm_create(const char *name, struct rte_cryptodev *dev; struct aesni_gcm_private *internals; enum aesni_gcm_vector_mode vector_mode; + MB_MGR *mb_mgr; /* Check CPU for support for AES instruction set */ if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { @@ -503,7 +504,9 @@ aesni_gcm_create(const char *name, } /* Check CPU for supported vector instruction set */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) + vector_mode = RTE_AESNI_GCM_AVX512; + else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) vector_mode = RTE_AESNI_GCM_AVX2; else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) vector_mode = RTE_AESNI_GCM_AVX; @@ -523,23 +526,64 @@ aesni_gcm_create(const char *name, RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + mb_mgr = alloc_mb_mgr(0); + if (mb_mgr == NULL) + return -ENOMEM; + switch (vector_mode) { case RTE_AESNI_GCM_SSE: dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + init_mb_mgr_sse(mb_mgr); break; case RTE_AESNI_GCM_AVX: dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + init_mb_mgr_avx(mb_mgr); break; case RTE_AESNI_GCM_AVX2: dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx2(mb_mgr); break; - default: + case RTE_AESNI_GCM_AVX512: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx512(mb_mgr); break; + default: + AESNI_GCM_LOG(ERR, "Unsupported vector mode %u\n", vector_mode); + goto error_exit; } internals = dev->data->dev_private; internals->vector_mode = vector_mode; + internals->mb_mgr = mb_mgr; + + /* Set arch independent function pointers, based on key size */ + internals->ops[GCM_KEY_128].enc = mb_mgr->gcm128_enc; + internals->ops[GCM_KEY_128].dec = mb_mgr->gcm128_dec; + internals->ops[GCM_KEY_128].pre = mb_mgr->gcm128_pre; + internals->ops[GCM_KEY_128].init = mb_mgr->gcm128_init; + internals->ops[GCM_KEY_128].update_enc = mb_mgr->gcm128_enc_update; + internals->ops[GCM_KEY_128].update_dec = mb_mgr->gcm128_dec_update; + internals->ops[GCM_KEY_128].finalize_enc = mb_mgr->gcm128_enc_finalize; + internals->ops[GCM_KEY_128].finalize_dec = mb_mgr->gcm128_dec_finalize; + + internals->ops[GCM_KEY_192].enc = mb_mgr->gcm192_enc; + internals->ops[GCM_KEY_192].dec = mb_mgr->gcm192_dec; + internals->ops[GCM_KEY_192].pre = mb_mgr->gcm192_pre; + internals->ops[GCM_KEY_192].init = mb_mgr->gcm192_init; + internals->ops[GCM_KEY_192].update_enc = mb_mgr->gcm192_enc_update; + internals->ops[GCM_KEY_192].update_dec = mb_mgr->gcm192_dec_update; + internals->ops[GCM_KEY_192].finalize_enc = mb_mgr->gcm192_enc_finalize; + internals->ops[GCM_KEY_192].finalize_dec = mb_mgr->gcm192_dec_finalize; + + internals->ops[GCM_KEY_256].enc = mb_mgr->gcm256_enc; + internals->ops[GCM_KEY_256].dec = mb_mgr->gcm256_dec; + internals->ops[GCM_KEY_256].pre = mb_mgr->gcm256_pre; + internals->ops[GCM_KEY_256].init = mb_mgr->gcm256_init; + internals->ops[GCM_KEY_256].update_enc = mb_mgr->gcm256_enc_update; + internals->ops[GCM_KEY_256].update_dec = mb_mgr->gcm256_dec_update; + internals->ops[GCM_KEY_256].finalize_enc = mb_mgr->gcm256_enc_finalize; + internals->ops[GCM_KEY_256].finalize_dec = mb_mgr->gcm256_dec_finalize; internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; @@ -551,6 +595,14 @@ aesni_gcm_create(const char *name, #endif return 0; + +error_exit: + if (mb_mgr) + free_mb_mgr(mb_mgr); + + rte_cryptodev_pmd_destroy(dev); + + return -1; } static int @@ -578,6 +630,7 @@ static int aesni_gcm_remove(struct rte_vdev_device *vdev) { struct rte_cryptodev *cryptodev; + struct aesni_gcm_private *internals; const char *name; name = rte_vdev_device_name(vdev); @@ -588,6 +641,10 @@ aesni_gcm_remove(struct rte_vdev_device *vdev) if (cryptodev == NULL) return -ENODEV; + internals = cryptodev->data->dev_private; + + free_mb_mgr(internals->mb_mgr); + return rte_cryptodev_pmd_destroy(cryptodev); } diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c index bf69dde54a..2f66c7c58e 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c @@ -227,7 +227,7 @@ aesni_gcm_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (aesni_gcm_pmd_qp_set_unique_name(dev, qp)) goto qp_setup_cleanup; - qp->ops = (const struct aesni_gcm_ops *)gcm_ops[internals->vector_mode]; + qp->ops = (const struct aesni_gcm_ops *)internals->ops; qp->processed_pkts = aesni_gcm_pmd_qp_create_processed_pkts_ring(qp, qp_conf->nb_descriptors, socket_id); @@ -283,7 +283,7 @@ aesni_gcm_pmd_sym_session_configure(struct rte_cryptodev *dev __rte_unused, "Couldn't get object from session mempool"); return -ENOMEM; } - ret = aesni_gcm_set_session_parameters(gcm_ops[internals->vector_mode], + ret = aesni_gcm_set_session_parameters(internals->ops, sess_private_data, xform); if (ret != 0) { AESNI_GCM_LOG(ERR, "failed configure session parameters"); diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h index 903e7503d3..56b29e013b 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h @@ -35,11 +35,15 @@ struct aesni_gcm_private { /**< Vector mode */ unsigned max_nb_queue_pairs; /**< Max number of queue pairs supported by device */ + MB_MGR *mb_mgr; + /**< Multi-buffer instance */ + struct aesni_gcm_ops ops[GCM_KEY_NUM]; + /**< Function pointer table of the gcm APIs */ }; struct aesni_gcm_qp { const struct aesni_gcm_ops *ops; - /**< Architecture dependent function pointer table of the gcm APIs */ + /**< Function pointer table of the gcm APIs */ struct rte_ring *processed_pkts; /**< Ring for placing process packets */ struct gcm_context_data gdata_ctx; /* (16 * 5) + 8 = 88 B */ -- 2.20.1