crypto/dpaa_sec: fix IOVA conversions

DPAA sec driver is using virtual to physical address
translation in its data path and driver is using
dpaax_iova_table_update() API in every address translation
which is very costly.
This patch moves dpaax_iova_table_update() calling to rte_dpaa_mem_ptov(),
only if it fails to found translation from DPAAX table.

Fixes: 12e5842945bf ("crypto/dpaa_sec: fix IOVA table")
Cc: stable@dpdk.org
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
Acked-by: Akhil Goyal <akhil.goyal@nxp.com>

diff --git a/drivers/bus/dpaa/rte_dpaa_bus.h b/drivers/bus/dpaa/rte_dpaa_bus.h
index 9bf2cd9..373aca9 100644 (file)
--- a/drivers/bus/dpaa/rte_dpaa_bus.h
+++ b/drivers/bus/dpaa/rte_dpaa_bus.h
@@ -132,7 +132,23 @@ static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr)
        }
 
        /* If not, Fallback to full memseg list searching */
-       return rte_mem_iova2virt(paddr);
+       va = rte_mem_iova2virt(paddr);
+
+       dpaax_iova_table_update(paddr, va, RTE_CACHE_LINE_SIZE);
+
+       return va;
+}
+
+static inline rte_iova_t
+rte_dpaa_mem_vtop(void *vaddr)
+{
+       const struct rte_memseg *ms;
+
+       ms = rte_mem_virt2memseg(vaddr, NULL);
+       if (ms)
+               return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
+
+       return (size_t)NULL;
 }
 
 /**

diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c
index e0b307c..df684d2 100644 (file)
--- a/drivers/crypto/dpaa_sec/dpaa_sec.c
+++ b/drivers/crypto/dpaa_sec/dpaa_sec.c
@@ -94,31 +94,6 @@ dpaa_sec_alloc_ctx(dpaa_sec_session *ses, int sg_count)
        return ctx;
 }
 
-static inline rte_iova_t
-dpaa_mem_vtop(void *vaddr)
-{
-       const struct rte_memseg *ms;
-
-       ms = rte_mem_virt2memseg(vaddr, NULL);
-       if (ms) {
-               dpaax_iova_table_update(ms->iova, ms->addr, ms->len);
-               return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
-       }
-       return (size_t)NULL;
-}
-
-static inline void *
-dpaa_mem_ptov(rte_iova_t paddr)
-{
-       void *va;
-
-       va = (void *)dpaax_iova_table_get_va(paddr);
-       if (likely(va))
-               return va;
-
-       return rte_mem_iova2virt(paddr);
-}
-
 static void
 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
                   struct qman_fq *fq,
@@ -183,7 +158,7 @@ dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
         * sg[0] is for output
         * sg[1] for input
         */
-       job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
+       job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 
        ctx = container_of(job, struct dpaa_sec_op_ctx, job);
        ctx->fd_status = fd->status;
@@ -304,12 +279,12 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses)
 
        if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) {
                cipherdata.key =
-                       (size_t)dpaa_mem_vtop((void *)(size_t)cipherdata.key);
+                       (size_t)rte_dpaa_mem_vtop((void *)(size_t)cipherdata.key);
                cipherdata.key_type = RTA_DATA_PTR;
        }
        if (!(cdb->sh_desc[2] & (1 << 1)) &&  authdata.keylen) {
                authdata.key =
-                       (size_t)dpaa_mem_vtop((void *)(size_t)authdata.key);
+                       (size_t)rte_dpaa_mem_vtop((void *)(size_t)authdata.key);
                authdata.key_type = RTA_DATA_PTR;
        }
 
@@ -405,14 +380,14 @@ dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses)
        if (cdb->sh_desc[2] & 1)
                cipherdata.key_type = RTA_DATA_IMM;
        else {
-               cipherdata.key = (size_t)dpaa_mem_vtop(
+               cipherdata.key = (size_t)rte_dpaa_mem_vtop(
                                        (void *)(size_t)cipherdata.key);
                cipherdata.key_type = RTA_DATA_PTR;
        }
        if (cdb->sh_desc[2] & (1<<1))
                authdata.key_type = RTA_DATA_IMM;
        else {
-               authdata.key = (size_t)dpaa_mem_vtop(
+               authdata.key = (size_t)rte_dpaa_mem_vtop(
                                        (void *)(size_t)authdata.key);
                authdata.key_type = RTA_DATA_PTR;
        }
@@ -591,14 +566,14 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses)
                if (cdb->sh_desc[2] & 1)
                        alginfo_c.key_type = RTA_DATA_IMM;
                else {
-                       alginfo_c.key = (size_t)dpaa_mem_vtop(
+                       alginfo_c.key = (size_t)rte_dpaa_mem_vtop(
                                                (void *)(size_t)alginfo_c.key);
                        alginfo_c.key_type = RTA_DATA_PTR;
                }
                if (cdb->sh_desc[2] & (1<<1))
                        alginfo_a.key_type = RTA_DATA_IMM;
                else {
-                       alginfo_a.key = (size_t)dpaa_mem_vtop(
+                       alginfo_a.key = (size_t)rte_dpaa_mem_vtop(
                                                (void *)(size_t)alginfo_a.key);
                        alginfo_a.key_type = RTA_DATA_PTR;
                }
@@ -674,7 +649,7 @@ dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
                 * sg[0] is for output
                 * sg[1] for input
                 */
-               job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
+               job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 
                ctx = container_of(job, struct dpaa_sec_op_ctx, job);
                ctx->fd_status = fd->status;
@@ -768,7 +743,7 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
        in_sg->extension = 1;
        in_sg->final = 1;
        in_sg->length = data_len;
-       qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2]));
+       qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
 
        /* 1st seg */
        sg = in_sg + 1;
@@ -788,7 +763,7 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
                } else {
                        sg->length = ses->iv.length;
                }
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(iv_ptr));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr));
                in_sg->length += sg->length;
                cpu_to_hw_sg(sg);
                sg++;
@@ -821,7 +796,7 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
                sg++;
                rte_memcpy(old_digest, sym->auth.digest.data,
                                ses->digest_length);
-               start_addr = dpaa_mem_vtop(old_digest);
+               start_addr = rte_dpaa_mem_vtop(old_digest);
                qm_sg_entry_set64(sg, start_addr);
                sg->length = ses->digest_length;
                in_sg->length += ses->digest_length;
@@ -888,7 +863,7 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
        in_sg->extension = 1;
        in_sg->final = 1;
        in_sg->length = data_len;
-       qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2]));
+       qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
        sg = &cf->sg[2];
 
        if (ses->iv.length) {
@@ -906,7 +881,7 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
                } else {
                        sg->length = ses->iv.length;
                }
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(iv_ptr));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr));
                in_sg->length += sg->length;
                cpu_to_hw_sg(sg);
                sg++;
@@ -923,7 +898,7 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
                rte_memcpy(old_digest, sym->auth.digest.data,
                                ses->digest_length);
                /* let's check digest by hw */
-               start_addr = dpaa_mem_vtop(old_digest);
+               start_addr = rte_dpaa_mem_vtop(old_digest);
                sg++;
                qm_sg_entry_set64(sg, start_addr);
                sg->length = ses->digest_length;
@@ -987,7 +962,7 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
        out_sg = &cf->sg[0];
        out_sg->extension = 1;
        out_sg->length = data_len;
-       qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2]));
+       qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
        cpu_to_hw_sg(out_sg);
 
        /* 1st seg */
@@ -1016,11 +991,11 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
        in_sg->length = data_len + ses->iv.length;
 
        sg++;
-       qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
        cpu_to_hw_sg(in_sg);
 
        /* IV */
-       qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
+       qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
        sg->length = ses->iv.length;
        cpu_to_hw_sg(sg);
 
@@ -1098,11 +1073,11 @@ build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
        sg->extension = 1;
        sg->final = 1;
        sg->length = data_len + ses->iv.length;
-       qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
+       qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(&cf->sg[2]));
        cpu_to_hw_sg(sg);
 
        sg = &cf->sg[2];
-       qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
+       qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
        sg->length = ses->iv.length;
        cpu_to_hw_sg(sg);
 
@@ -1163,7 +1138,7 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 
        /* output sg entries */
        sg = &cf->sg[2];
-       qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
        cpu_to_hw_sg(out_sg);
 
        /* 1st seg */
@@ -1206,18 +1181,18 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 
        /* input sg entries */
        sg++;
-       qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
        cpu_to_hw_sg(in_sg);
 
        /* 1st seg IV */
-       qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
+       qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
        sg->length = ses->iv.length;
        cpu_to_hw_sg(sg);
 
        /* 2nd seg auth only */
        if (ses->auth_only_len) {
                sg++;
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(sym->aead.aad.data));
                sg->length = ses->auth_only_len;
                cpu_to_hw_sg(sg);
        }
@@ -1243,7 +1218,7 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
                sg++;
                memcpy(ctx->digest, sym->aead.digest.data,
                        ses->digest_length);
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
                sg->length = ses->digest_length;
        }
        sg->final = 1;
@@ -1281,9 +1256,9 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
        /* input */
        rte_prefetch0(cf->sg);
        sg = &cf->sg[2];
-       qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg));
        if (is_encode(ses)) {
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
                sg->length = ses->iv.length;
                length += sg->length;
                cpu_to_hw_sg(sg);
@@ -1291,7 +1266,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
                sg++;
                if (ses->auth_only_len) {
                        qm_sg_entry_set64(sg,
-                                         dpaa_mem_vtop(sym->aead.aad.data));
+                                         rte_dpaa_mem_vtop(sym->aead.aad.data));
                        sg->length = ses->auth_only_len;
                        length += sg->length;
                        cpu_to_hw_sg(sg);
@@ -1303,7 +1278,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
                sg->final = 1;
                cpu_to_hw_sg(sg);
        } else {
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
                sg->length = ses->iv.length;
                length += sg->length;
                cpu_to_hw_sg(sg);
@@ -1311,7 +1286,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
                sg++;
                if (ses->auth_only_len) {
                        qm_sg_entry_set64(sg,
-                                         dpaa_mem_vtop(sym->aead.aad.data));
+                                         rte_dpaa_mem_vtop(sym->aead.aad.data));
                        sg->length = ses->auth_only_len;
                        length += sg->length;
                        cpu_to_hw_sg(sg);
@@ -1326,7 +1301,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
                       ses->digest_length);
                sg++;
 
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
                sg->length = ses->digest_length;
                length += sg->length;
                sg->final = 1;
@@ -1340,7 +1315,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
 
        /* output */
        sg++;
-       qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg));
        qm_sg_entry_set64(sg,
                dst_start_addr + sym->aead.data.offset);
        sg->length = sym->aead.data.length;
@@ -1409,7 +1384,7 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 
        /* output sg entries */
        sg = &cf->sg[2];
-       qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
        cpu_to_hw_sg(out_sg);
 
        /* 1st seg */
@@ -1451,11 +1426,11 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 
        /* input sg entries */
        sg++;
-       qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
        cpu_to_hw_sg(in_sg);
 
        /* 1st seg IV */
-       qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
+       qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
        sg->length = ses->iv.length;
        cpu_to_hw_sg(sg);
 
@@ -1481,7 +1456,7 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
                sg++;
                memcpy(ctx->digest, sym->auth.digest.data,
                        ses->digest_length);
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
                sg->length = ses->digest_length;
        }
        sg->final = 1;
@@ -1518,9 +1493,9 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
        /* input */
        rte_prefetch0(cf->sg);
        sg = &cf->sg[2];
-       qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg));
        if (is_encode(ses)) {
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
                sg->length = ses->iv.length;
                length += sg->length;
                cpu_to_hw_sg(sg);
@@ -1532,7 +1507,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
                sg->final = 1;
                cpu_to_hw_sg(sg);
        } else {
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
                sg->length = ses->iv.length;
                length += sg->length;
                cpu_to_hw_sg(sg);
@@ -1548,7 +1523,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
                       ses->digest_length);
                sg++;
 
-               qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
+               qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
                sg->length = ses->digest_length;
                length += sg->length;
                sg->final = 1;
@@ -1562,7 +1537,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
 
        /* output */
        sg++;
-       qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg));
        qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
        sg->length = sym->cipher.data.length;
        length = sg->length;
@@ -1656,7 +1631,7 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
        /* output */
        out_sg = &cf->sg[0];
        out_sg->extension = 1;
-       qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2]));
+       qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
 
        /* 1st seg */
        sg = &cf->sg[2];
@@ -1689,7 +1664,7 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
        in_len = mbuf->data_len;
 
        sg++;
-       qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
+       qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
 
        /* 1st seg */
        qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
@@ -1884,7 +1859,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
                        inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES];
                        fd->opaque_addr = 0;
                        fd->cmd = 0;
-                       qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg));
+                       qm_fd_addr_set64(fd, rte_dpaa_mem_vtop(cf->sg));
                        fd->_format1 = qm_fd_compound;
                        fd->length29 = 2 * sizeof(struct qm_sg_entry);
 
@@ -2349,7 +2324,7 @@ dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
                }
        }
        ret = dpaa_sec_init_rx(sess->inq[rte_lcore_id() % MAX_DPAA_CORES],
-                              dpaa_mem_vtop(&sess->cdb),
+                              rte_dpaa_mem_vtop(&sess->cdb),
                               qman_fq_fqid(&qp->outq));
        if (ret)
                DPAA_SEC_ERR("Unable to init sec queue");
@@ -3149,7 +3124,7 @@ dpaa_sec_process_parallel_event(void *event,
         * sg[0] is for output
         * sg[1] for input
         */
-       job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
+       job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 
        ctx = container_of(job, struct dpaa_sec_op_ctx, job);
        ctx->fd_status = fd->status;
@@ -3204,7 +3179,7 @@ dpaa_sec_process_atomic_event(void *event,
         * sg[0] is for output
         * sg[1] for input
         */
-       job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
+       job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 
        ctx = container_of(job, struct dpaa_sec_op_ctx, job);
        ctx->fd_status = fd->status;

diff --git a/drivers/crypto/dpaa_sec/meson.build b/drivers/crypto/dpaa_sec/meson.build
index 9f17d3a..71de819 100644 (file)
--- a/drivers/crypto/dpaa_sec/meson.build
+++ b/drivers/crypto/dpaa_sec/meson.build
@@ -6,7 +6,7 @@ if not is_linux
        reason = 'only supported on linux'
 endif
 
-deps += ['bus_dpaa', 'security']
+deps += ['bus_dpaa', 'mempool_dpaa', 'security']
 sources = files('dpaa_sec.c')
 
 allow_experimental_apis = true