crypto/dpaa_sec: affine the thread portal affinity

[dpdk.git] / drivers / crypto / dpaa_sec / dpaa_sec.c

diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c
index 0495422..19d4684 100644 (file)
--- a/drivers/crypto/dpaa_sec/dpaa_sec.c
+++ b/drivers/crypto/dpaa_sec/dpaa_sec.c
@@ -15,11 +15,12 @@
 #include <rte_cryptodev_pmd.h>
 #include <rte_crypto.h>
 #include <rte_cryptodev.h>
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
 #include <rte_security_driver.h>
 #endif
 #include <rte_cycles.h>
 #include <rte_dev.h>
+#include <rte_ip.h>
 #include <rte_kvargs.h>
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
@@ -36,6 +37,7 @@
 #include <desc/algo.h>
 #include <desc/ipsec.h>
 #include <desc/pdcp.h>
+#include <desc/sdap.h>
 
 #include <rte_dpaa_bus.h>
 #include <dpaa_sec.h>
@@ -229,7 +231,7 @@ static inline int is_decode(dpaa_sec_session *ses)
        return ses->dir == DIR_DEC;
 }
 
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
 static int
 dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses)
 {
@@ -293,24 +295,49 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses)
                                        &cipherdata, &authdata,
                                        0);
        } else {
-               if (ses->dir == DIR_ENC)
-                       shared_desc_len = cnstr_shdsc_pdcp_u_plane_encap(
-                                       cdb->sh_desc, 1, swap,
-                                       ses->pdcp.sn_size,
-                                       ses->pdcp.hfn,
-                                       ses->pdcp.bearer,
-                                       ses->pdcp.pkt_dir,
-                                       ses->pdcp.hfn_threshold,
-                                       &cipherdata, p_authdata, 0);
-               else if (ses->dir == DIR_DEC)
-                       shared_desc_len = cnstr_shdsc_pdcp_u_plane_decap(
-                                       cdb->sh_desc, 1, swap,
-                                       ses->pdcp.sn_size,
-                                       ses->pdcp.hfn,
-                                       ses->pdcp.bearer,
-                                       ses->pdcp.pkt_dir,
-                                       ses->pdcp.hfn_threshold,
-                                       &cipherdata, p_authdata, 0);
+               if (ses->dir == DIR_ENC) {
+                       if (ses->pdcp.sdap_enabled)
+                               shared_desc_len =
+                                       cnstr_shdsc_pdcp_sdap_u_plane_encap(
+                                               cdb->sh_desc, 1, swap,
+                                               ses->pdcp.sn_size,
+                                               ses->pdcp.hfn,
+                                               ses->pdcp.bearer,
+                                               ses->pdcp.pkt_dir,
+                                               ses->pdcp.hfn_threshold,
+                                               &cipherdata, p_authdata, 0);
+                       else
+                               shared_desc_len =
+                                       cnstr_shdsc_pdcp_u_plane_encap(
+                                               cdb->sh_desc, 1, swap,
+                                               ses->pdcp.sn_size,
+                                               ses->pdcp.hfn,
+                                               ses->pdcp.bearer,
+                                               ses->pdcp.pkt_dir,
+                                               ses->pdcp.hfn_threshold,
+                                               &cipherdata, p_authdata, 0);
+               } else if (ses->dir == DIR_DEC) {
+                       if (ses->pdcp.sdap_enabled)
+                               shared_desc_len =
+                                       cnstr_shdsc_pdcp_sdap_u_plane_decap(
+                                               cdb->sh_desc, 1, swap,
+                                               ses->pdcp.sn_size,
+                                               ses->pdcp.hfn,
+                                               ses->pdcp.bearer,
+                                               ses->pdcp.pkt_dir,
+                                               ses->pdcp.hfn_threshold,
+                                               &cipherdata, p_authdata, 0);
+                       else
+                               shared_desc_len =
+                                       cnstr_shdsc_pdcp_u_plane_decap(
+                                               cdb->sh_desc, 1, swap,
+                                               ses->pdcp.sn_size,
+                                               ses->pdcp.hfn,
+                                               ses->pdcp.bearer,
+                                               ses->pdcp.pkt_dir,
+                                               ses->pdcp.hfn_threshold,
+                                               &cipherdata, p_authdata, 0);
+               }
        }
        return shared_desc_len;
 }
@@ -408,7 +435,7 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses)
        memset(cdb, 0, sizeof(struct sec_cdb));
 
        switch (ses->ctxt) {
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
        case DPAA_SEC_IPSEC:
                shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses);
                break;
@@ -1539,7 +1566,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
        return cf;
 }
 
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
 static inline struct dpaa_sec_job *
 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
 {
@@ -1690,13 +1717,20 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
        uint32_t index, flags[DPAA_SEC_BURST] = {0};
        struct qman_fq *inq[DPAA_SEC_BURST];
 
+       if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
+               if (rte_dpaa_portal_init((void *)0)) {
+                       DPAA_SEC_ERR("Failure in affining portal");
+                       return 0;
+               }
+       }
+
        while (nb_ops) {
                frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
                                DPAA_SEC_BURST : nb_ops;
                for (loop = 0; loop < frames_to_send; loop++) {
                        op = *(ops++);
-                       if (op->sym->m_src->seqn != 0) {
-                               index = op->sym->m_src->seqn - 1;
+                       if (*dpaa_seqn(op->sym->m_src) != 0) {
+                               index = *dpaa_seqn(op->sym->m_src) - 1;
                                if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) {
                                        /* QM_EQCR_DCA_IDXMASK = 0x0f */
                                        flags[loop] = ((index & 0x0f) << 8);
@@ -1714,7 +1748,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
                                                        op->sym->session,
                                                        cryptodev_driver_id);
                                break;
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
                        case RTE_CRYPTO_OP_SECURITY_SESSION:
                                ses = (dpaa_sec_session *)
                                        get_sec_session_private_data(
@@ -1761,7 +1795,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
                                  ((op->sym->m_dst == NULL) ||
                                   rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
                                switch (ses->ctxt) {
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
                                case DPAA_SEC_PDCP:
                                case DPAA_SEC_IPSEC:
                                        cf = build_proto(op, ses);
@@ -1795,7 +1829,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
                                }
                        } else {
                                switch (ses->ctxt) {
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
                                case DPAA_SEC_PDCP:
                                case DPAA_SEC_IPSEC:
                                        cf = build_proto_sg(op, ses);
@@ -1852,7 +1886,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
                                        ((auth_tail_len << 16) | auth_hdr_len);
                        }
 
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
                        /* In case of PDCP, per packet HFN is stored in
                         * mbuf priv after sym_op.
                         */
@@ -1890,6 +1924,13 @@ dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
        uint16_t num_rx;
        struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
 
+       if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
+               if (rte_dpaa_portal_init((void *)0)) {
+                       DPAA_SEC_ERR("Failure in affining portal");
+                       return 0;
+               }
+       }
+
        num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
 
        dpaa_qp->rx_pkts += num_rx;
@@ -2257,7 +2298,7 @@ dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
        for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) {
                if (&qi->inq[i] == fq) {
                        if (qman_retire_fq(fq, NULL) != 0)
-                               DPAA_SEC_WARN("Queue is not retired\n");
+                               DPAA_SEC_DEBUG("Queue is not retired\n");
                        qman_oos_fq(fq);
                        qi->inq_attach[i] = 0;
                        return 0;
@@ -2467,7 +2508,7 @@ dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
        }
 }
 
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
 static int
 dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
                        struct rte_security_ipsec_xform *ipsec_xform,
@@ -2949,6 +2990,7 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev,
        session->pdcp.hfn = pdcp_xform->hfn;
        session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
        session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
+       session->pdcp.sdap_enabled = pdcp_xform->sdap_enabled;
        if (cipher_xform)
                session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
 
@@ -3185,7 +3227,7 @@ dpaa_sec_process_atomic_event(void *event,
        DPAA_PER_LCORE_DQRR_HELD |= 1 << index;
        DPAA_PER_LCORE_DQRR_MBUF(index) = ctx->op->sym->m_src;
        ev->impl_opaque = index + 1;
-       ctx->op->sym->m_src->seqn = (uint32_t)index + 1;
+       *dpaa_seqn(ctx->op->sym->m_src) = (uint32_t)index + 1;
        *bufs = (void *)ctx->op;
 
        rte_mempool_put(ctx->ctx_pool, (void *)ctx);
@@ -3272,7 +3314,7 @@ static struct rte_cryptodev_ops crypto_ops = {
        .sym_session_clear        = dpaa_sec_sym_session_clear
 };
 
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
 static const struct rte_security_capability *
 dpaa_sec_capabilities_get(void *device __rte_unused)
 {
@@ -3311,7 +3353,7 @@ static int
 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
 {
        struct dpaa_sec_dev_private *internals;
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
        struct rte_security_ctx *security_instance;
 #endif
        struct dpaa_sec_qp *qp;
@@ -3348,7 +3390,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
                DPAA_SEC_WARN("Device already init by primary process");
                return 0;
        }
-#ifdef RTE_LIBRTE_SECURITY
+#ifdef RTE_LIB_SECURITY
        /* Initialize security_ctx only for primary process*/
        security_instance = rte_malloc("rte_security_instances_ops",
                                sizeof(struct rte_security_ctx), 0);