crypto/cnxk: account for CPT CTX updates and flush delays

CPT CTX write with microcode would require CPT flush to complete to have
DRAM updated with the SA. Since datapath requires SA direction field,
introduce a new flag for the same.

Session destroy path is also updated to clear sa.valid bit using CTX
reload operation.

Session is updated with marker to differentiate s/w immutable and s/w
mutable portions.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index d25a17c..7617bdc 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -53,7 +53,6 @@ cpt_sec_inst_fill(struct rte_crypto_op *op, struct cn10k_sec_session *sess,
                  struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst)
 {
        struct rte_crypto_sym_op *sym_op = op->sym;
-       union roc_ot_ipsec_sa_word2 *w2;
        struct cn10k_ipsec_sa *sa;
        int ret;
 
@@ -68,9 +67,8 @@ cpt_sec_inst_fill(struct rte_crypto_op *op, struct cn10k_sec_session *sess,
        }
 
        sa = &sess->sa;
-       w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
 
-       if (w2->s.dir == ROC_IE_SA_DIR_OUTBOUND)
+       if (sa->is_outbound)
                ret = process_outb_sa(op, sa, inst);
        else {
                infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;

diff --git a/drivers/crypto/cnxk/cn10k_ipsec.c b/drivers/crypto/cnxk/cn10k_ipsec.c
index a11a6b7..b4acbac 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_ipsec.c
+++ b/drivers/crypto/cnxk/cn10k_ipsec.c
@@ -67,7 +67,7 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
                goto sa_dptr_free;
        }
 
-       sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, sa);
+       sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, out_sa);
 
 #ifdef LA_IPSEC_DEBUG
        /* Use IV from application in debug mode */
@@ -89,6 +89,8 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
        }
 #endif
 
+       sa->is_outbound = true;
+
        /* Get Rlen calculation data */
        ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm);
        if (ret)
@@ -127,6 +129,8 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
        /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
        memcpy(out_sa, sa_dptr, 8);
 
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
        /* Write session using microcode opcode */
        ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa,
                                ROC_NIX_INL_OT_IPSEC_OUTB_HW_SZ);
@@ -135,9 +139,11 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
                goto sa_dptr_free;
        }
 
-       /* Trigger CTX flush to write dirty data back to DRAM */
+       /* Trigger CTX flush so that data is written back to DRAM */
        roc_cpt_lf_ctx_flush(lf, out_sa, false);
 
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
 sa_dptr_free:
        plt_free(sa_dptr);
 
@@ -178,7 +184,8 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
                goto sa_dptr_free;
        }
 
-       sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, sa);
+       sa->is_outbound = false;
+       sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, in_sa);
 
        /* pre-populate CPT INST word 4 */
        inst_w4.u64 = 0;
@@ -214,6 +221,8 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
        /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
        memcpy(in_sa, sa_dptr, 8);
 
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
        /* Write session using microcode opcode */
        ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa,
                                ROC_NIX_INL_OT_IPSEC_INB_HW_SZ);
@@ -222,9 +231,11 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
                goto sa_dptr_free;
        }
 
-       /* Trigger CTX flush to write dirty data back to DRAM */
+       /* Trigger CTX flush so that data is written back to DRAM */
        roc_cpt_lf_ctx_flush(lf, in_sa, false);
 
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
 sa_dptr_free:
        plt_free(sa_dptr);
 
@@ -300,21 +311,46 @@ mempool_put:
 }
 
 static int
-cn10k_sec_session_destroy(void *device __rte_unused,
-                         struct rte_security_session *sess)
+cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
 {
-       struct cn10k_sec_session *priv;
+       struct rte_cryptodev *crypto_dev = dev;
+       union roc_ot_ipsec_sa_word2 *w2;
+       struct cn10k_sec_session *sess;
        struct rte_mempool *sess_mp;
+       struct cn10k_ipsec_sa *sa;
+       struct cnxk_cpt_qp *qp;
+       struct roc_cpt_lf *lf;
 
-       priv = get_sec_session_private_data(sess);
+       sess = get_sec_session_private_data(sec_sess);
+       if (sess == NULL)
+               return 0;
 
-       if (priv == NULL)
+       qp = crypto_dev->data->queue_pairs[0];
+       if (qp == NULL)
                return 0;
 
-       sess_mp = rte_mempool_from_obj(priv);
+       lf = &qp->lf;
 
-       set_sec_session_private_data(sess, NULL);
-       rte_mempool_put(sess_mp, priv);
+       sa = &sess->sa;
+
+       /* Trigger CTX flush to write dirty data back to DRAM */
+       roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false);
+
+       /* Wait for 1 ms so that flush is complete */
+       rte_delay_ms(1);
+
+       w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
+       w2->s.valid = 0;
+
+       plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+       /* Trigger CTX reload to fetch new data from DRAM */
+       roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
+
+       sess_mp = rte_mempool_from_obj(sess);
+
+       set_sec_session_private_data(sec_sess, NULL);
+       rte_mempool_put(sess_mp, sess);
 
        return 0;
 }

diff --git a/drivers/crypto/cnxk/cn10k_ipsec.h b/drivers/crypto/cnxk/cn10k_ipsec.h
index 86cd248..cc7ca19 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_ipsec.h
+++ b/drivers/crypto/cnxk/cn10k_ipsec.h
@@ -7,28 +7,37 @@
 
 #include <rte_security.h>
 
+#include "roc_api.h"
+
 #include "cnxk_ipsec.h"
 
-#define CN10K_IPSEC_SA_CTX_HDR_SIZE 1
+typedef void *CN10K_SA_CONTEXT_MARKER[0];
 
 struct cn10k_ipsec_sa {
-       union {
-               /** Inbound SA */
-               struct roc_ot_ipsec_inb_sa in_sa;
-               /** Outbound SA */
-               struct roc_ot_ipsec_outb_sa out_sa;
-       };
        /** Pre-populated CPT inst words */
        struct cnxk_cpt_inst_tmpl inst;
        uint16_t max_extended_len;
        uint16_t iv_offset;
        uint8_t iv_length;
        bool ip_csum_enable;
-};
+       bool is_outbound;
+
+       /**
+        * End of SW mutable area
+        */
+       CN10K_SA_CONTEXT_MARKER sw_area_end __rte_aligned(ROC_ALIGN);
+
+       union {
+               /** Inbound SA */
+               struct roc_ot_ipsec_inb_sa in_sa;
+               /** Outbound SA */
+               struct roc_ot_ipsec_outb_sa out_sa;
+       };
+} __rte_aligned(ROC_ALIGN);
 
 struct cn10k_sec_session {
        struct cn10k_ipsec_sa sa;
-} __rte_cache_aligned;
+} __rte_aligned(ROC_ALIGN);
 
 void cn10k_sec_ops_override(void);
 

diff --git a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
index 881fbd1..cab6a50 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
@@ -54,6 +54,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sess,
        struct rte_crypto_sym_op *sym_op = cop->sym;
        struct rte_mbuf *m_src = sym_op->m_src;
        uint64_t inst_w4_u64 = sess->inst.w4;
+       uint64_t dptr;
 
        if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) {
                plt_dp_err("Not enough tail room");
@@ -76,10 +77,10 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sess,
                inst_w4_u64 &= ~BIT_ULL(32);
 
        /* Prepare CPT instruction */
-       inst->w4.u64 = inst_w4_u64;
-       inst->w4.s.dlen = rte_pktmbuf_pkt_len(m_src);
-       inst->dptr = rte_pktmbuf_iova(m_src);
-       inst->rptr = inst->dptr;
+       inst->w4.u64 = inst_w4_u64 | rte_pktmbuf_pkt_len(m_src);
+       dptr = rte_pktmbuf_iova(m_src);
+       inst->dptr = dptr;
+       inst->rptr = dptr;
 
        return 0;
 }
@@ -90,12 +91,13 @@ process_inb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sa,
 {
        struct rte_crypto_sym_op *sym_op = cop->sym;
        struct rte_mbuf *m_src = sym_op->m_src;
+       uint64_t dptr;
 
        /* Prepare CPT instruction */
-       inst->w4.u64 = sa->inst.w4;
-       inst->w4.s.dlen = rte_pktmbuf_pkt_len(m_src);
-       inst->dptr = rte_pktmbuf_iova(m_src);
-       inst->rptr = inst->dptr;
+       inst->w4.u64 = sa->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+       dptr = rte_pktmbuf_iova(m_src);
+       inst->dptr = dptr;
+       inst->rptr = dptr;
 
        return 0;
 }