From 568e6db2203ee4675f795b3f59a0f809e5e81c57 Mon Sep 17 00:00:00 2001 From: Anoob Joseph Date: Fri, 17 Dec 2021 14:49:55 +0530 Subject: [PATCH] crypto/cnxk: account for CPT CTX updates and flush delays CPT CTX write with microcode would require CPT flush to complete to have DRAM updated with the SA. Since datapath requires SA direction field, introduce a new flag for the same. Session destroy path is also updated to clear sa.valid bit using CTX reload operation. Session is updated with marker to differentiate s/w immutable and s/w mutable portions. Signed-off-by: Anoob Joseph Acked-by: Akhil Goyal --- drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 4 +- drivers/crypto/cnxk/cn10k_ipsec.c | 60 ++++++++++++++++++----- drivers/crypto/cnxk/cn10k_ipsec.h | 27 ++++++---- drivers/crypto/cnxk/cn10k_ipsec_la_ops.h | 18 ++++--- 4 files changed, 77 insertions(+), 32 deletions(-) diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c index d25a17cbf3..7617bdcd87 100644 --- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c +++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c @@ -53,7 +53,6 @@ cpt_sec_inst_fill(struct rte_crypto_op *op, struct cn10k_sec_session *sess, struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst) { struct rte_crypto_sym_op *sym_op = op->sym; - union roc_ot_ipsec_sa_word2 *w2; struct cn10k_ipsec_sa *sa; int ret; @@ -68,9 +67,8 @@ cpt_sec_inst_fill(struct rte_crypto_op *op, struct cn10k_sec_session *sess, } sa = &sess->sa; - w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2; - if (w2->s.dir == ROC_IE_SA_DIR_OUTBOUND) + if (sa->is_outbound) ret = process_outb_sa(op, sa, inst); else { infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND; diff --git a/drivers/crypto/cnxk/cn10k_ipsec.c b/drivers/crypto/cnxk/cn10k_ipsec.c index a11a6b7d4c..b4acbaca31 100644 --- a/drivers/crypto/cnxk/cn10k_ipsec.c +++ b/drivers/crypto/cnxk/cn10k_ipsec.c @@ -67,7 +67,7 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, goto sa_dptr_free; } - sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, sa); + sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, out_sa); #ifdef LA_IPSEC_DEBUG /* Use IV from application in debug mode */ @@ -89,6 +89,8 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, } #endif + sa->is_outbound = true; + /* Get Rlen calculation data */ ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm); if (ret) @@ -127,6 +129,8 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */ memcpy(out_sa, sa_dptr, 8); + plt_atomic_thread_fence(__ATOMIC_SEQ_CST); + /* Write session using microcode opcode */ ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa, ROC_NIX_INL_OT_IPSEC_OUTB_HW_SZ); @@ -135,9 +139,11 @@ cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, goto sa_dptr_free; } - /* Trigger CTX flush to write dirty data back to DRAM */ + /* Trigger CTX flush so that data is written back to DRAM */ roc_cpt_lf_ctx_flush(lf, out_sa, false); + plt_atomic_thread_fence(__ATOMIC_SEQ_CST); + sa_dptr_free: plt_free(sa_dptr); @@ -178,7 +184,8 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, goto sa_dptr_free; } - sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, sa); + sa->is_outbound = false; + sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, in_sa); /* pre-populate CPT INST word 4 */ inst_w4.u64 = 0; @@ -214,6 +221,8 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */ memcpy(in_sa, sa_dptr, 8); + plt_atomic_thread_fence(__ATOMIC_SEQ_CST); + /* Write session using microcode opcode */ ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa, ROC_NIX_INL_OT_IPSEC_INB_HW_SZ); @@ -222,9 +231,11 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf, goto sa_dptr_free; } - /* Trigger CTX flush to write dirty data back to DRAM */ + /* Trigger CTX flush so that data is written back to DRAM */ roc_cpt_lf_ctx_flush(lf, in_sa, false); + plt_atomic_thread_fence(__ATOMIC_SEQ_CST); + sa_dptr_free: plt_free(sa_dptr); @@ -300,21 +311,46 @@ mempool_put: } static int -cn10k_sec_session_destroy(void *device __rte_unused, - struct rte_security_session *sess) +cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess) { - struct cn10k_sec_session *priv; + struct rte_cryptodev *crypto_dev = dev; + union roc_ot_ipsec_sa_word2 *w2; + struct cn10k_sec_session *sess; struct rte_mempool *sess_mp; + struct cn10k_ipsec_sa *sa; + struct cnxk_cpt_qp *qp; + struct roc_cpt_lf *lf; - priv = get_sec_session_private_data(sess); + sess = get_sec_session_private_data(sec_sess); + if (sess == NULL) + return 0; - if (priv == NULL) + qp = crypto_dev->data->queue_pairs[0]; + if (qp == NULL) return 0; - sess_mp = rte_mempool_from_obj(priv); + lf = &qp->lf; - set_sec_session_private_data(sess, NULL); - rte_mempool_put(sess_mp, priv); + sa = &sess->sa; + + /* Trigger CTX flush to write dirty data back to DRAM */ + roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false); + + /* Wait for 1 ms so that flush is complete */ + rte_delay_ms(1); + + w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2; + w2->s.valid = 0; + + plt_atomic_thread_fence(__ATOMIC_SEQ_CST); + + /* Trigger CTX reload to fetch new data from DRAM */ + roc_cpt_lf_ctx_reload(lf, &sa->in_sa); + + sess_mp = rte_mempool_from_obj(sess); + + set_sec_session_private_data(sec_sess, NULL); + rte_mempool_put(sess_mp, sess); return 0; } diff --git a/drivers/crypto/cnxk/cn10k_ipsec.h b/drivers/crypto/cnxk/cn10k_ipsec.h index 86cd2483f5..cc7ca19865 100644 --- a/drivers/crypto/cnxk/cn10k_ipsec.h +++ b/drivers/crypto/cnxk/cn10k_ipsec.h @@ -7,28 +7,37 @@ #include +#include "roc_api.h" + #include "cnxk_ipsec.h" -#define CN10K_IPSEC_SA_CTX_HDR_SIZE 1 +typedef void *CN10K_SA_CONTEXT_MARKER[0]; struct cn10k_ipsec_sa { - union { - /** Inbound SA */ - struct roc_ot_ipsec_inb_sa in_sa; - /** Outbound SA */ - struct roc_ot_ipsec_outb_sa out_sa; - }; /** Pre-populated CPT inst words */ struct cnxk_cpt_inst_tmpl inst; uint16_t max_extended_len; uint16_t iv_offset; uint8_t iv_length; bool ip_csum_enable; -}; + bool is_outbound; + + /** + * End of SW mutable area + */ + CN10K_SA_CONTEXT_MARKER sw_area_end __rte_aligned(ROC_ALIGN); + + union { + /** Inbound SA */ + struct roc_ot_ipsec_inb_sa in_sa; + /** Outbound SA */ + struct roc_ot_ipsec_outb_sa out_sa; + }; +} __rte_aligned(ROC_ALIGN); struct cn10k_sec_session { struct cn10k_ipsec_sa sa; -} __rte_cache_aligned; +} __rte_aligned(ROC_ALIGN); void cn10k_sec_ops_override(void); diff --git a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h index 881fbd19b3..cab6a5047c 100644 --- a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h +++ b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h @@ -54,6 +54,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sess, struct rte_crypto_sym_op *sym_op = cop->sym; struct rte_mbuf *m_src = sym_op->m_src; uint64_t inst_w4_u64 = sess->inst.w4; + uint64_t dptr; if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) { plt_dp_err("Not enough tail room"); @@ -76,10 +77,10 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sess, inst_w4_u64 &= ~BIT_ULL(32); /* Prepare CPT instruction */ - inst->w4.u64 = inst_w4_u64; - inst->w4.s.dlen = rte_pktmbuf_pkt_len(m_src); - inst->dptr = rte_pktmbuf_iova(m_src); - inst->rptr = inst->dptr; + inst->w4.u64 = inst_w4_u64 | rte_pktmbuf_pkt_len(m_src); + dptr = rte_pktmbuf_iova(m_src); + inst->dptr = dptr; + inst->rptr = dptr; return 0; } @@ -90,12 +91,13 @@ process_inb_sa(struct rte_crypto_op *cop, struct cn10k_ipsec_sa *sa, { struct rte_crypto_sym_op *sym_op = cop->sym; struct rte_mbuf *m_src = sym_op->m_src; + uint64_t dptr; /* Prepare CPT instruction */ - inst->w4.u64 = sa->inst.w4; - inst->w4.s.dlen = rte_pktmbuf_pkt_len(m_src); - inst->dptr = rte_pktmbuf_iova(m_src); - inst->rptr = inst->dptr; + inst->w4.u64 = sa->inst.w4 | rte_pktmbuf_pkt_len(m_src); + dptr = rte_pktmbuf_iova(m_src); + inst->dptr = dptr; + inst->rptr = dptr; return 0; } -- 2.20.1