crypto/aesni_gcm: support SGL on AES-GMAC

Add Scatter-gather list support for AES-GMAC.

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
Tested-by: Fan Zhang <roy.fan.zhang@intel.com>
Acked-by: Fan Zhang <roy.fan.zhang@intel.com>

diff --git a/doc/guides/cryptodevs/aesni_gcm.rst b/doc/guides/cryptodevs/aesni_gcm.rst
index 74e0de6..0e14648 100644 (file)
--- a/doc/guides/cryptodevs/aesni_gcm.rst
+++ b/doc/guides/cryptodevs/aesni_gcm.rst
@@ -31,8 +31,6 @@ Limitations
 -----------
 
 * In out-of-place operations, chained destination mbufs are not supported.
-* Chained mbufs are only supported by RTE_CRYPTO_AEAD_AES_GCM algorithm,
-  not RTE_CRYPTO_AUTH_AES_GMAC.
 * Cipher only is not supported.
 
 

diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index 954be3f..80217a0 100644 (file)
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -159,6 +159,10 @@ New Features
   * Added support for SNOW3G-UEA2/UIA2 algorithms.
   * Added support for KASUMI-F8/F9 algorithms.
 
+* **Updated the aesni_gcm crypto PMD.**
+
+  * Added SGL support for AES-GMAC.
+
 * **Added Broadcom BCMFS symmetric crypto PMD.**
 
   Added a symmetric crypto PMD for Broadcom FlexSparc crypto units.

diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h
index 74acac0..8a0d074 100644 (file)
--- a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h
+++ b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h
@@ -53,6 +53,23 @@ typedef void (*aesni_gcm_finalize_t)(const struct gcm_key_data *gcm_key_data,
                uint8_t *auth_tag,
                uint64_t auth_tag_len);
 
+#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM
+typedef void (*aesni_gmac_init_t)(const struct gcm_key_data *gcm_key_data,
+               struct gcm_context_data *gcm_ctx_data,
+               const uint8_t *iv,
+               const uint64_t iv_len);
+
+typedef void (*aesni_gmac_update_t)(const struct gcm_key_data *gcm_key_data,
+               struct gcm_context_data *gcm_ctx_data,
+               const uint8_t *in,
+               const uint64_t plaintext_len);
+
+typedef void (*aesni_gmac_finalize_t)(const struct gcm_key_data *gcm_key_data,
+               struct gcm_context_data *gcm_ctx_data,
+               uint8_t *auth_tag,
+               const uint64_t auth_tag_len);
+#endif
+
 /** GCM library function pointer table */
 struct aesni_gcm_ops {
        aesni_gcm_t enc;        /**< GCM encode function pointer */
@@ -63,6 +80,11 @@ struct aesni_gcm_ops {
        aesni_gcm_update_t update_dec;
        aesni_gcm_finalize_t finalize_enc;
        aesni_gcm_finalize_t finalize_dec;
+#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM
+       aesni_gmac_init_t gmac_init;
+       aesni_gmac_update_t gmac_update;
+       aesni_gmac_finalize_t gmac_finalize;
+#endif
 };
 
 /** GCM per-session operation handlers */
@@ -72,6 +94,11 @@ struct aesni_gcm_session_ops {
        aesni_gcm_init_t init;
        aesni_gcm_update_t update;
        aesni_gcm_finalize_t finalize;
+#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM
+       aesni_gmac_init_t gmac_init;
+       aesni_gmac_update_t gmac_update;
+       aesni_gmac_finalize_t gmac_finalize;
+#endif
 };
 
 #endif /* _AESNI_GCM_OPS_H_ */

diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
index 1d2a0ce..aea599e 100644 (file)
--- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
+++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c
@@ -350,6 +350,76 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op,
                                &qp->gdata_ctx,
                                tag,
                                session->gen_digest_length);
+#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM
+       } else if (session->op == AESNI_GMAC_OP_GENERATE) {
+               qp->ops[session->key].gmac_init(&session->gdata_key,
+                               &qp->gdata_ctx,
+                               iv_ptr,
+                               session->iv.length);
+
+               qp->ops[session->key].gmac_update(&session->gdata_key,
+                               &qp->gdata_ctx, src,
+                               (uint64_t)part_len);
+               total_len = data_length - part_len;
+
+               while (total_len) {
+                       m_src = m_src->next;
+
+                       RTE_ASSERT(m_src != NULL);
+
+                       src = rte_pktmbuf_mtod(m_src, uint8_t *);
+                       part_len = (m_src->data_len < total_len) ?
+                                       m_src->data_len : total_len;
+
+                       qp->ops[session->key].gmac_update(&session->gdata_key,
+                                       &qp->gdata_ctx, src,
+                                       (uint64_t)part_len);
+                       total_len -= part_len;
+               }
+
+               if (session->req_digest_length != session->gen_digest_length)
+                       tag = qp->temp_digest;
+               else
+                       tag = sym_op->auth.digest.data;
+
+               qp->ops[session->key].gmac_finalize(&session->gdata_key,
+                               &qp->gdata_ctx,
+                               tag,
+                               session->gen_digest_length);
+       } else { /* AESNI_GMAC_OP_VERIFY */
+               qp->ops[session->key].gmac_init(&session->gdata_key,
+                               &qp->gdata_ctx,
+                               iv_ptr,
+                               session->iv.length);
+
+               qp->ops[session->key].gmac_update(&session->gdata_key,
+                               &qp->gdata_ctx, src,
+                               (uint64_t)part_len);
+               total_len = data_length - part_len;
+
+               while (total_len) {
+                       m_src = m_src->next;
+
+                       RTE_ASSERT(m_src != NULL);
+
+                       src = rte_pktmbuf_mtod(m_src, uint8_t *);
+                       part_len = (m_src->data_len < total_len) ?
+                                       m_src->data_len : total_len;
+
+                       qp->ops[session->key].gmac_update(&session->gdata_key,
+                                       &qp->gdata_ctx, src,
+                                       (uint64_t)part_len);
+                       total_len -= part_len;
+               }
+
+               tag = qp->temp_digest;
+
+               qp->ops[session->key].gmac_finalize(&session->gdata_key,
+                               &qp->gdata_ctx,
+                               tag,
+                               session->gen_digest_length);
+       }
+#else
        } else if (session->op == AESNI_GMAC_OP_GENERATE) {
                qp->ops[session->key].init(&session->gdata_key,
                                &qp->gdata_ctx,
@@ -381,6 +451,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op,
                                tag,
                                session->gen_digest_length);
        }
+#endif
 
        return 0;
 }
@@ -769,7 +840,7 @@ aesni_gcm_create(const char *name,
                init_mb_mgr_avx2(mb_mgr);
                break;
        case RTE_AESNI_GCM_AVX512:
-               dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
+               dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
                init_mb_mgr_avx512(mb_mgr);
                break;
        default:
@@ -791,6 +862,11 @@ aesni_gcm_create(const char *name,
        internals->ops[GCM_KEY_128].update_dec = mb_mgr->gcm128_dec_update;
        internals->ops[GCM_KEY_128].finalize_enc = mb_mgr->gcm128_enc_finalize;
        internals->ops[GCM_KEY_128].finalize_dec = mb_mgr->gcm128_dec_finalize;
+#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM
+       internals->ops[GCM_KEY_128].gmac_init = mb_mgr->gmac128_init;
+       internals->ops[GCM_KEY_128].gmac_update = mb_mgr->gmac128_update;
+       internals->ops[GCM_KEY_128].gmac_finalize = mb_mgr->gmac128_finalize;
+#endif
 
        internals->ops[GCM_KEY_192].enc = mb_mgr->gcm192_enc;
        internals->ops[GCM_KEY_192].dec = mb_mgr->gcm192_dec;
@@ -800,6 +876,11 @@ aesni_gcm_create(const char *name,
        internals->ops[GCM_KEY_192].update_dec = mb_mgr->gcm192_dec_update;
        internals->ops[GCM_KEY_192].finalize_enc = mb_mgr->gcm192_enc_finalize;
        internals->ops[GCM_KEY_192].finalize_dec = mb_mgr->gcm192_dec_finalize;
+#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM
+       internals->ops[GCM_KEY_192].gmac_init = mb_mgr->gmac192_init;
+       internals->ops[GCM_KEY_192].gmac_update = mb_mgr->gmac192_update;
+       internals->ops[GCM_KEY_192].gmac_finalize = mb_mgr->gmac192_finalize;
+#endif
 
        internals->ops[GCM_KEY_256].enc = mb_mgr->gcm256_enc;
        internals->ops[GCM_KEY_256].dec = mb_mgr->gcm256_dec;
@@ -809,6 +890,11 @@ aesni_gcm_create(const char *name,
        internals->ops[GCM_KEY_256].update_dec = mb_mgr->gcm256_dec_update;
        internals->ops[GCM_KEY_256].finalize_enc = mb_mgr->gcm256_enc_finalize;
        internals->ops[GCM_KEY_256].finalize_dec = mb_mgr->gcm256_dec_finalize;
+#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM
+       internals->ops[GCM_KEY_256].gmac_init = mb_mgr->gmac256_init;
+       internals->ops[GCM_KEY_256].gmac_update = mb_mgr->gmac256_update;
+       internals->ops[GCM_KEY_256].gmac_finalize = mb_mgr->gmac256_finalize;
+#endif
 
        internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;