net/bonding: prefer allmulti to promiscuous for LACP

[dpdk.git] / drivers / crypto / qat / qat_sym.c

diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c
index 17d63eb..46ef27a 100644 (file)
--- a/drivers/crypto/qat/qat_sym.c
+++ b/drivers/crypto/qat/qat_sym.c
@@ -156,7 +156,10 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
        uint32_t auth_len = 0, auth_ofs = 0;
        uint32_t min_ofs = 0;
        uint64_t src_buf_start = 0, dst_buf_start = 0;
+       uint64_t auth_data_end = 0;
        uint8_t do_sgl = 0;
+       uint8_t in_place = 1;
+       int alignment_adjustment = 0;
        struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
        struct qat_sym_op_cookie *cookie =
                                (struct qat_sym_op_cookie *)op_cookie;
@@ -174,7 +177,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
                return -EINVAL;
        }
 
-       ctx = (struct qat_sym_session *)get_session_private_data(
+       ctx = (struct qat_sym_session *)get_sym_session_private_data(
                        op->sym->session, cryptodev_qat_driver_id);
 
        if (unlikely(ctx == NULL)) {
@@ -225,9 +228,8 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
                                ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
 
                        if (unlikely(
-                               (cipher_param->cipher_length % BYTE_LENGTH != 0)
-                                || (cipher_param->cipher_offset
-                                                       % BYTE_LENGTH != 0))) {
+                           (op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
+                           (op->sym->cipher.data.offset % BYTE_LENGTH != 0))) {
                                QAT_DP_LOG(ERR,
                  "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
                                op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
@@ -260,8 +262,9 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
                        ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
                        ctx->qat_hash_alg ==
                                ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
-                       if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0)
-                               || (auth_param->auth_len % BYTE_LENGTH != 0))) {
+                       if (unlikely(
+                           (op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
+                           (op->sym->auth.data.length % BYTE_LENGTH != 0))) {
                                QAT_DP_LOG(ERR,
                "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
                                op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
@@ -438,6 +441,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
                 * Don't align DMA start. DMA the minimum data-set
                 * so as not to overwrite data in dest buffer
                 */
+               in_place = 0;
                src_buf_start =
                        rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs);
                dst_buf_start =
@@ -462,6 +466,10 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
                                                                min_ofs);
                }
                dst_buf_start = src_buf_start;
+
+               /* remember any adjustment for later, note, can be +/- */
+               alignment_adjustment = src_buf_start -
+                       rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs);
        }
 
        if (do_cipher || do_aead) {
@@ -490,13 +498,66 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
                (cipher_param->cipher_offset + cipher_param->cipher_length)
                : (auth_param->auth_off + auth_param->auth_len);
 
+       if (do_auth && do_cipher) {
+               /* Handle digest-encrypted cases, i.e.
+                * auth-gen-then-cipher-encrypt and
+                * cipher-decrypt-then-auth-verify
+                */
+                /* First find the end of the data */
+               if (do_sgl) {
+                       uint32_t remaining_off = auth_param->auth_off +
+                               auth_param->auth_len + alignment_adjustment;
+                       struct rte_mbuf *sgl_buf =
+                               (in_place ?
+                                       op->sym->m_src : op->sym->m_dst);
+
+                       while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
+                                       && sgl_buf->next != NULL) {
+                               remaining_off -= rte_pktmbuf_data_len(sgl_buf);
+                               sgl_buf = sgl_buf->next;
+                       }
+
+                       auth_data_end = (uint64_t)rte_pktmbuf_iova_offset(
+                               sgl_buf, remaining_off);
+               } else {
+                       auth_data_end = (in_place ?
+                               src_buf_start : dst_buf_start) +
+                               auth_param->auth_off + auth_param->auth_len;
+               }
+               /* Then check if digest-encrypted conditions are met */
+               if ((auth_param->auth_off + auth_param->auth_len <
+                                       cipher_param->cipher_offset +
+                                       cipher_param->cipher_length) &&
+                               (op->sym->auth.digest.phys_addr ==
+                                       auth_data_end)) {
+                       /* Handle partial digest encryption */
+                       if (cipher_param->cipher_offset +
+                                       cipher_param->cipher_length <
+                                       auth_param->auth_off +
+                                       auth_param->auth_len +
+                                       ctx->digest_length)
+                               qat_req->comn_mid.dst_length =
+                                       qat_req->comn_mid.src_length =
+                                       auth_param->auth_off +
+                                       auth_param->auth_len +
+                                       ctx->digest_length;
+                       struct icp_qat_fw_comn_req_hdr *header =
+                               &qat_req->comn_hdr;
+                       ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
+                               header->serv_specif_flags,
+                               ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+               }
+       }
+
        if (do_sgl) {
 
                ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags,
                                QAT_COMN_PTR_TYPE_SGL);
-               ret = qat_sgl_fill_array(op->sym->m_src, src_buf_start,
-                               &cookie->qat_sgl_src,
-                               qat_req->comn_mid.src_length);
+               ret = qat_sgl_fill_array(op->sym->m_src,
+                  (int64_t)(src_buf_start - rte_pktmbuf_iova(op->sym->m_src)),
+                  &cookie->qat_sgl_src,
+                  qat_req->comn_mid.src_length,
+                  QAT_SYM_SGL_MAX_NUMBER);
 
                if (unlikely(ret)) {
                        QAT_DP_LOG(ERR, "QAT PMD Cannot fill sgl array");
@@ -509,9 +570,11 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
                                cookie->qat_sgl_src_phys_addr;
                else {
                        ret = qat_sgl_fill_array(op->sym->m_dst,
-                                       dst_buf_start,
-                                       &cookie->qat_sgl_dst,
-                                               qat_req->comn_mid.dst_length);
+                               (int64_t)(dst_buf_start -
+                                         rte_pktmbuf_iova(op->sym->m_dst)),
+                                &cookie->qat_sgl_dst,
+                                qat_req->comn_mid.dst_length,
+                                QAT_SYM_SGL_MAX_NUMBER);
 
                        if (unlikely(ret)) {
                                QAT_DP_LOG(ERR, "QAT PMD can't fill sgl array");
@@ -523,6 +586,8 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
                        qat_req->comn_mid.dest_data_addr =
                                        cookie->qat_sgl_dst_phys_addr;
                }
+               qat_req->comn_mid.src_length = 0;
+               qat_req->comn_mid.dst_length = 0;
        } else {
                qat_req->comn_mid.src_data_addr = src_buf_start;
                qat_req->comn_mid.dest_data_addr = dst_buf_start;