crypto/ipsec_mb: support snow3g digest appended ops

This patch enables out-of-place auth-cipher operations where
digest should be encrypted along with the rest of raw data.
It also adds support for partially encrypted digest when using
auth-cipher operations.

Signed-off-by: Damian Nowak <damianx.nowak@intel.com>
Signed-off-by: Kai Ji <kai.ji@intel.com>
Signed-off-by: Piotr Bronowski <piotrx.bronowski@intel.com>
Signed-off-by: Ciara Power <ciara.power@intel.com>
Acked-by: Ray Kinsella <mdr@ashroe.eu>
Acked-by: Fan Zhang <roy.fan.zhang@intel.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>

diff --git a/doc/guides/cryptodevs/features/snow3g.ini b/doc/guides/cryptodevs/features/snow3g.ini
index 14ac7e4..4d4c5b5 100644 (file)
--- a/doc/guides/cryptodevs/features/snow3g.ini
+++ b/doc/guides/cryptodevs/features/snow3g.ini
@@ -8,6 +8,7 @@ Symmetric crypto       = Y
 Sym operation chaining = Y
 Symmetric sessionless  = Y
 Non-Byte aligned data  = Y
+Digest encrypted       = Y
 OOP LB  In LB  Out     = Y
 
 ;

diff --git a/doc/guides/rel_notes/release_21_11.rst b/doc/guides/rel_notes/release_21_11.rst
index feee38c..fecc721 100644 (file)
--- a/doc/guides/rel_notes/release_21_11.rst
+++ b/doc/guides/rel_notes/release_21_11.rst
@@ -239,6 +239,13 @@ New Features
 
   * Added support for ZUC-EEA3-256 and ZUC-EIA3-256.
 
+* **Added digest appended ops support for Snow3G PMD.**
+
+  * Added support for out-of-place auth-cipher operations that encrypt
+    the digest along with the rest of the raw data.
+  * Added support for partially encrypted digest when using auth-cipher
+    operations.
+
 * **Updated the ACC100 bbdev PMD.**
 
   Added support for more comprehensive CRC options.

diff --git a/drivers/crypto/ipsec_mb/pmd_snow3g.c b/drivers/crypto/ipsec_mb/pmd_snow3g.c
index 3b76c60..ebc9a0b 100644 (file)
--- a/drivers/crypto/ipsec_mb/pmd_snow3g.c
+++ b/drivers/crypto/ipsec_mb/pmd_snow3g.c
@@ -73,6 +73,24 @@ snow3g_session_configure(IMB_MGR *mgr, void *priv_sess,
        return 0;
 }
 
+/** Check if conditions are met for digest-appended operations */
+static uint8_t *
+snow3g_digest_appended_in_src(struct rte_crypto_op *op)
+{
+       unsigned int auth_size, cipher_size;
+
+       auth_size = (op->sym->auth.data.offset >> 3) +
+               (op->sym->auth.data.length >> 3);
+       cipher_size = (op->sym->cipher.data.offset >> 3) +
+               (op->sym->cipher.data.length >> 3);
+
+       if (auth_size < cipher_size)
+               return rte_pktmbuf_mtod_offset(op->sym->m_src,
+                               uint8_t *, auth_size);
+
+       return NULL;
+}
+
 /** Encrypt/decrypt mbufs with same cipher key. */
 static uint8_t
 process_snow3g_cipher_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
@@ -81,29 +99,75 @@ process_snow3g_cipher_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
 {
        uint32_t i;
        uint8_t processed_ops = 0;
-       const void *src[SNOW3G_MAX_BURST];
-       void *dst[SNOW3G_MAX_BURST];
-       const void *iv[SNOW3G_MAX_BURST];
-       uint32_t num_bytes[SNOW3G_MAX_BURST];
+       const void *src[SNOW3G_MAX_BURST] = {NULL};
+       void *dst[SNOW3G_MAX_BURST] = {NULL};
+       uint8_t *digest_appended[SNOW3G_MAX_BURST] = {NULL};
+       const void *iv[SNOW3G_MAX_BURST] = {NULL};
+       uint32_t num_bytes[SNOW3G_MAX_BURST] = {0};
+       uint32_t cipher_off, cipher_len;
+       int unencrypted_bytes = 0;
 
        for (i = 0; i < num_ops; i++) {
-               src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
-                               (ops[i]->sym->cipher.data.offset >> 3);
-               dst[i] = ops[i]->sym->m_dst ?
-                       rte_pktmbuf_mtod(ops[i]->sym->m_dst, uint8_t *) +
-                               (ops[i]->sym->cipher.data.offset >> 3) :
-                       rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
-                               (ops[i]->sym->cipher.data.offset >> 3);
+
+               cipher_off = ops[i]->sym->cipher.data.offset >> 3;
+               cipher_len = ops[i]->sym->cipher.data.length >> 3;
+               src[i] = rte_pktmbuf_mtod_offset(
+                       ops[i]->sym->m_src,     uint8_t *, cipher_off);
+
+               /* If out-of-place operation */
+               if (ops[i]->sym->m_dst &&
+                       ops[i]->sym->m_src != ops[i]->sym->m_dst) {
+                       dst[i] = rte_pktmbuf_mtod_offset(
+                               ops[i]->sym->m_dst, uint8_t *, cipher_off);
+
+                       /* In case of out-of-place, auth-cipher operation
+                        * with partial encryption of the digest, copy
+                        * the remaining, unencrypted part.
+                        */
+                       if (session->op == IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT
+                           || session->op == IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT)
+                               unencrypted_bytes =
+                                       (ops[i]->sym->auth.data.offset >> 3) +
+                                       (ops[i]->sym->auth.data.length >> 3) +
+                                       (SNOW3G_DIGEST_LENGTH) -
+                                       cipher_off - cipher_len;
+                       if (unencrypted_bytes > 0)
+                               rte_memcpy(
+                                       rte_pktmbuf_mtod_offset(
+                                               ops[i]->sym->m_dst, uint8_t *,
+                                               cipher_off + cipher_len),
+                                       rte_pktmbuf_mtod_offset(
+                                               ops[i]->sym->m_src, uint8_t *,
+                                               cipher_off + cipher_len),
+                                       unencrypted_bytes);
+               } else
+                       dst[i] = rte_pktmbuf_mtod_offset(ops[i]->sym->m_src,
+                                               uint8_t *, cipher_off);
+
                iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
                                session->cipher_iv_offset);
-               num_bytes[i] = ops[i]->sym->cipher.data.length >> 3;
-
+               num_bytes[i] = cipher_len;
                processed_ops++;
        }
 
        IMB_SNOW3G_F8_N_BUFFER(qp->mb_mgr, &session->pKeySched_cipher, iv,
                        src, dst, num_bytes, processed_ops);
 
+       /* Take care of the raw digest data in src buffer */
+       for (i = 0; i < num_ops; i++) {
+               if ((session->op == IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT ||
+                       session->op == IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT) &&
+                               ops[i]->sym->m_dst != NULL) {
+                       digest_appended[i] =
+                               snow3g_digest_appended_in_src(ops[i]);
+                       /* Clear unencrypted digest from
+                        * the src buffer
+                        */
+                       if (digest_appended[i] != NULL)
+                               memset(digest_appended[i],
+                                       0, SNOW3G_DIGEST_LENGTH);
+               }
+       }
        return processed_ops;
 }
 
@@ -116,6 +180,7 @@ process_snow3g_cipher_op_bit(struct ipsec_mb_qp *qp,
        uint8_t *src, *dst;
        uint8_t *iv;
        uint32_t length_in_bits, offset_in_bits;
+       int unencrypted_bytes = 0;
 
        offset_in_bits = op->sym->cipher.data.offset;
        src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *);
@@ -124,10 +189,32 @@ process_snow3g_cipher_op_bit(struct ipsec_mb_qp *qp,
                IPSEC_MB_LOG(ERR, "bit-level in-place not supported\n");
                return 0;
        }
+       length_in_bits = op->sym->cipher.data.length;
        dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *);
+       /* In case of out-of-place, auth-cipher operation
+        * with partial encryption of the digest, copy
+        * the remaining, unencrypted part.
+        */
+       if (session->op == IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT ||
+               session->op == IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT)
+               unencrypted_bytes =
+                       (op->sym->auth.data.offset >> 3) +
+                       (op->sym->auth.data.length >> 3) +
+                       (SNOW3G_DIGEST_LENGTH) -
+                       (offset_in_bits >> 3) -
+                       (length_in_bits >> 3);
+       if (unencrypted_bytes > 0)
+               rte_memcpy(
+                       rte_pktmbuf_mtod_offset(
+                               op->sym->m_dst, uint8_t *,
+                               (length_in_bits >> 3)),
+                       rte_pktmbuf_mtod_offset(
+                               op->sym->m_src, uint8_t *,
+                               (length_in_bits >> 3)),
+                               unencrypted_bytes);
+
        iv = rte_crypto_op_ctod_offset(op, uint8_t *,
                                session->cipher_iv_offset);
-       length_in_bits = op->sym->cipher.data.length;
 
        IMB_SNOW3G_F8_1_BUFFER_BIT(qp->mb_mgr, &session->pKeySched_cipher, iv,
                        src, dst, length_in_bits, offset_in_bits);
@@ -146,6 +233,7 @@ process_snow3g_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
        uint8_t *src, *dst;
        uint32_t length_in_bits;
        uint8_t *iv;
+       uint8_t digest_appended = 0;
        struct snow3g_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
 
        for (i = 0; i < num_ops; i++) {
@@ -156,6 +244,8 @@ process_snow3g_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
                        break;
                }
 
+               dst = NULL;
+
                length_in_bits = ops[i]->sym->auth.data.length;
 
                src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
@@ -165,6 +255,15 @@ process_snow3g_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
 
                if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) {
                        dst = qp_data->temp_digest;
+                        /* Handle auth cipher verify oop case*/
+                       if ((session->op ==
+                               IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN ||
+                               session->op ==
+                               IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY) &&
+                               ops[i]->sym->m_dst != NULL)
+                               src = rte_pktmbuf_mtod_offset(
+                                       ops[i]->sym->m_dst, uint8_t *,
+                                       ops[i]->sym->auth.data.offset >> 3);
 
                        IMB_SNOW3G_F9_1_BUFFER(qp->mb_mgr,
                                        &session->pKeySched_hash,
@@ -174,12 +273,26 @@ process_snow3g_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
                                        SNOW3G_DIGEST_LENGTH) != 0)
                                ops[i]->status =
                                        RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
-               } else  {
-                       dst = ops[i]->sym->auth.digest.data;
+               } else {
+                       if (session->op ==
+                               IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT ||
+                               session->op ==
+                               IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT)
+                               dst = snow3g_digest_appended_in_src(ops[i]);
+
+                       if (dst != NULL)
+                               digest_appended = 1;
+                       else
+                               dst = ops[i]->sym->auth.digest.data;
 
                        IMB_SNOW3G_F9_1_BUFFER(qp->mb_mgr,
                                        &session->pKeySched_hash,
                                        iv, src, length_in_bits, dst);
+
+                       /* Copy back digest from src to auth.digest.data */
+                       if (digest_appended)
+                               rte_memcpy(ops[i]->sym->auth.digest.data,
+                                       dst, SNOW3G_DIGEST_LENGTH);
                }
                processed_ops++;
        }
@@ -480,7 +593,8 @@ RTE_INIT(ipsec_mb_register_snow3g)
                        RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
                        RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
                        RTE_CRYPTODEV_FF_SYM_SESSIONLESS |
-                       RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+                       RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
+                       RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED;
        snow3g_data->internals_priv_size = 0;
        snow3g_data->ops = &snow3g_pmd_ops;
        snow3g_data->qp_priv_size = sizeof(struct snow3g_qp_data);