crypto/cnxk: account for CPT CTX updates and flush delays

CPT CTX write with microcode would require CPT flush to complete to have
DRAM updated with the SA. Since datapath requires SA direction field,
introduce a new flag for the same.

Session destroy path is also updated to clear sa.valid bit using CTX
reload operation.

Session is updated with marker to differentiate s/w immutable and s/w
mutable portions.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index d25a17cbf333a3fc13c2e9c448c6349efec743ad..7617bdcd873faaf816bc7ca04f2572ba55d44a60 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -53,7 +53,6 @@ cpt_sec_inst_fill(struct rte_crypto_op *op, struct cn10k_sec_session *sess,
                  struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst)
 {
        struct rte_crypto_sym_op *sym_op = op->sym;
-       union roc_ot_ipsec_sa_word2 *w2;
        struct cn10k_ipsec_sa *sa;
        int ret;
 
@@ -68,9 +67,8 @@ cpt_sec_inst_fill(struct rte_crypto_op *op, struct cn10k_sec_session *sess,
        }
 
        sa = &sess->sa;
-       w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
 
-       if (w2->s.dir == ROC_IE_SA_DIR_OUTBOUND)
+       if (sa->is_outbound)
                ret = process_outb_sa(op, sa, inst);
        else {
                infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;

diff --git a/drivers/crypto/cnxk/cn10k_ipsec.c b/drivers/crypto/cnxk/cn10k_ipsec.c
index a11a6b7d4caa7048955de4b2aa3adef9c9a7f9b6..b4acbaca3190d2aad117e5c6fc30d7364e2b66b8 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_ipsec.c
+++ b/drivers/crypto/cnxk/cn10k_ipsec.c
@@ -67,7 +67,7 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
                goto sa_dptr_free;
        }
 
-       sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, sa);
+       sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, out_sa);
 
 #ifdef LA_IPSEC_DEBUG
        /* Use IV from application in debug mode */
@@ -89,6 +89,8 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
        }
 #endif
 
+       sa->is_outbound = true;
+
        /* Get Rlen calculation data */
        ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm);
        if (ret)
@@ -127,6 +129,8 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
        /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
        memcpy(out_sa, sa_dptr, 8);
 
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
        /* Write session using microcode opcode */
        ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa,
                                ROC_NIX_INL_OT_IPSEC_OUTB_HW_SZ);
@@ -135,9 +139,11 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
                goto sa_dptr_free;
        }
 
-       /* Trigger CTX flush to write dirty data back to DRAM */
+       /* Trigger CTX flush so that data is written back to DRAM */
        roc_cpt_lf_ctx_flush(lf, out_sa, false);
 
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
 sa_dptr_free:
        plt_free(sa_dptr);
 
@@ -178,7 +184,8 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
                goto sa_dptr_free;
        }
 
-       sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, sa);
+       sa->is_outbound = false;
+       sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, in_sa);
 
        /* pre-populate CPT INST word 4 */
        inst_w4.u64 = 0;
@@ -214,6 +221,8 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
        /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
        memcpy(in_sa, sa_dptr, 8);
 
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
        /* Write session using microcode opcode */
        ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa,
                                ROC_NIX_INL_OT_IPSEC_INB_HW_SZ);
@@ -222,9 +231,11 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
                goto sa_dptr_free;
        }
 
-       /* Trigger CTX flush to write dirty data back to DRAM */
+       /* Trigger CTX flush so that data is written back to DRAM */
        roc_cpt_lf_ctx_flush(lf, in_sa, false);
 
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
 sa_dptr_free:
        plt_free(sa_dptr);
 
@@ -300,21 +311,46 @@ mempool_put:
 }
 
 static int
-cn10k_sec_session_destroy(void *device __rte_unused,
-                         struct rte_security_session *sess)
+cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
 {
-       struct cn10k_sec_session *priv;
+       struct rte_cryptodev *crypto_dev = dev;
+       union roc_ot_ipsec_sa_word2 *w2;
+       struct cn10k_sec_session *sess;
        struct rte_mempool *sess_mp;
+       struct cn10k_ipsec_sa *sa;
+       struct cnxk_cpt_qp *qp;
+       struct roc_cpt_lf *lf;
 
-       priv = get_sec_session_private_data(sess);
+       sess = get_sec_session_private_data(sec_sess);
+       if (sess == NULL)
+               return 0;
 
-       if (priv == NULL)
+       qp = crypto_dev->data->queue_pairs[0];
+       if (qp == NULL)
                return 0;
 
-       sess_mp = rte_mempool_from_obj(priv);
+       lf = &qp->lf;
 
-       set_sec_session_private_data(sess, NULL);
-       rte_mempool_put(sess_mp, priv);
+       sa = &sess->sa;
+
+       /* Trigger CTX flush to write dirty data back to DRAM */
+       roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false);
+
+       /* Wait for 1 ms so that flush is complete */
+       rte_delay_ms(1);
+
+       w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
+       w2->s.valid = 0;
+
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+       /* Trigger CTX reload to fetch new data from DRAM */
+       roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
+
+       sess_mp = rte_mempool_from_obj(sess);
+
+       set_sec_session_private_data(sec_sess, NULL);
+       rte_mempool_put(sess_mp, sess);
 
        return 0;
 }

diff --git a/drivers/crypto/cnxk/cn10k_ipsec.h b/drivers/crypto/cnxk/cn10k_ipsec.h
index 86cd2483f548291e157978c754eb176915236394..cc7ca198650f04720b9052f56d6d885a55699926 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_ipsec.h
+++ b/drivers/crypto/cnxk/cn10k_ipsec.h
@@ -7,28 +7,37 @@
 
 #include <rte_security.h>
 
+#include "roc_api.h"
+
 #include "cnxk_ipsec.h"
 
-#define CN10K_IPSEC_SA_CTX_HDR_SIZE 1
+typedef void *CN10K_SA_CONTEXT_MARKER[0];
 
 struct cn10k_ipsec_sa {
-       union {
-               /** Inbound SA */
-               struct roc_ot_ipsec_inb_sa in_sa;
-               /** Outbound SA */
-               struct roc_ot_ipsec_outb_sa out_sa;
-       };
        /** Pre-populated CPT inst words */
        struct cnxk_cpt_inst_tmpl inst;
        uint16_t max_extended_len;
        uint16_t iv_offset;
        uint8_t iv_length;
        bool ip_csum_enable;
-};
+       bool is_outbound;
+
+       /**
+        * End of SW mutable area
+        */
+       CN10K_SA_CONTEXT_MARKER sw_area_end __rte_aligned(ROC_ALIGN);
+
+       union {
+               /** Inbound SA */
+               struct roc_ot_ipsec_inb_sa in_sa;
+               /** Outbound SA */
+               struct roc_ot_ipsec_outb_sa out_sa;
+       };
+} __rte_aligned(ROC_ALIGN);
 
 struct cn10k_sec_session {
        struct cn10k_ipsec_sa sa;
-} __rte_cache_aligned;
+} __rte_aligned(ROC_ALIGN);
 
 void cn10k_sec_ops_override(void);
 

diff --git a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
index 881fbd19b30c7b3240a22b8450e482215da4dfa6..cab6a5047ca02d908c46dc5a798add3f669b9f98 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
@@ -54,6 +54,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sess,
        struct rte_crypto_sym_op *sym_op = cop->sym;
        struct rte_mbuf *m_src = sym_op->m_src;
        uint64_t inst_w4_u64 = sess->inst.w4;
+       uint64_t dptr;
 
        if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) {
                plt_dp_err("Not enough tail room");
@@ -76,10 +77,10 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sess,
                inst_w4_u64 &= ~BIT_ULL(32);
 
        /* Prepare CPT instruction */
-       inst->w4.u64 = inst_w4_u64;
-       inst->w4.s.dlen = rte_pktmbuf_pkt_len(m_src);
-       inst->dptr = rte_pktmbuf_iova(m_src);
-       inst->rptr = inst->dptr;
+       inst->w4.u64 = inst_w4_u64 | rte_pktmbuf_pkt_len(m_src);
+       dptr = rte_pktmbuf_iova(m_src);
+       inst->dptr = dptr;
+       inst->rptr = dptr;
 
        return 0;
 }
@@ -90,12 +91,13 @@ process_inb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sa,
 {
        struct rte_crypto_sym_op *sym_op = cop->sym;
        struct rte_mbuf *m_src = sym_op->m_src;
+       uint64_t dptr;
 
        /* Prepare CPT instruction */
-       inst->w4.u64 = sa->inst.w4;
-       inst->w4.s.dlen = rte_pktmbuf_pkt_len(m_src);
-       inst->dptr = rte_pktmbuf_iova(m_src);
-       inst->rptr = inst->dptr;
+       inst->w4.u64 = sa->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+       dptr = rte_pktmbuf_iova(m_src);
+       inst->dptr = dptr;
+       inst->rptr = dptr;
 
        return 0;
 }