mbuf: add namespace to offload flags

[dpdk.git] / drivers / crypto / cnxk / cn10k_cryptodev_ops.c

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 3caf05a..d25a17c 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -50,7 +50,7 @@ sess_put:
 
 static __rte_always_inline int __rte_hot
 cpt_sec_inst_fill(struct rte_crypto_op *op, struct cn10k_sec_session *sess,
-                 struct cpt_inst_s *inst)
+                 struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst)
 {
        struct rte_crypto_sym_op *sym_op = op->sym;
        union roc_ot_ipsec_sa_word2 *w2;
@@ -72,8 +72,10 @@ cpt_sec_inst_fill(struct rte_crypto_op *op, struct cn10k_sec_session *sess,
 
        if (w2->s.dir == ROC_IE_SA_DIR_OUTBOUND)
                ret = process_outb_sa(op, sa, inst);
-       else
+       else {
+               infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
                ret = process_inb_sa(op, sa, inst);
+       }
 
        return ret;
 }
@@ -122,7 +124,8 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[],
                if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
                        sec_sess = get_sec_session_private_data(
                                sym_op->sec_session);
-                       ret = cpt_sec_inst_fill(op, sec_sess, &inst[0]);
+                       ret = cpt_sec_inst_fill(op, sec_sess, infl_req,
+                                               &inst[0]);
                        if (unlikely(ret))
                                return 0;
                        w7 = sec_sess->sa.inst.w7;
@@ -193,11 +196,15 @@ cn10k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
        struct pending_queue *pend_q;
        struct cpt_inst_s *inst;
        uint16_t lmt_id;
+       uint64_t head;
        int ret, i;
 
        pend_q = &qp->pend_q;
 
-       nb_allowed = qp->lf.nb_desc - pend_q->pending_count;
+       const uint64_t pq_mask = pend_q->pq_mask;
+
+       head = pend_q->head;
+       nb_allowed = pending_queue_free_cnt(head, pend_q->tail, pq_mask);
        nb_ops = RTE_MIN(nb_ops, nb_allowed);
 
        if (unlikely(nb_ops == 0))
@@ -211,18 +218,18 @@ cn10k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 
 again:
        for (i = 0; i < RTE_MIN(PKTS_PER_LOOP, nb_ops); i++) {
-               infl_req = &pend_q->req_queue[pend_q->enq_tail];
+               infl_req = &pend_q->req_queue[head];
                infl_req->op_flags = 0;
 
                ret = cn10k_cpt_fill_inst(qp, ops + i, &inst[2 * i], infl_req);
                if (unlikely(ret != 1)) {
                        plt_dp_err("Could not process op: %p", ops + i);
                        if (i == 0)
-                               goto update_pending;
+                               goto pend_q_commit;
                        break;
                }
 
-               MOD_INC(pend_q->enq_tail, qp->lf.nb_desc);
+               pending_queue_advance(&head, pq_mask);
        }
 
        if (i > PKTS_PER_STEORL) {
@@ -248,9 +255,10 @@ again:
                goto again;
        }
 
-update_pending:
-       pend_q->pending_count += count + i;
+pend_q_commit:
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
 
+       pend_q->head = head;
        pend_q->time_out = rte_get_timer_cycles() +
                           DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
 
@@ -342,6 +350,49 @@ cn10k_cpt_sec_post_process(struct rte_crypto_op *cop,
        m->pkt_len = m_len;
 }
 
+static inline void
+cn10k_cpt_sec_ucc_process(struct rte_crypto_op *cop,
+                         struct cpt_inflight_req *infl_req,
+                         const uint8_t uc_compcode)
+{
+       struct cn10k_sec_session *sess;
+       struct cn10k_ipsec_sa *sa;
+       struct rte_mbuf *mbuf;
+
+       if (uc_compcode == ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST)
+               cop->aux_flags = RTE_CRYPTO_OP_AUX_FLAGS_IPSEC_SOFT_EXPIRY;
+
+       if (!(infl_req->op_flags & CPT_OP_FLAGS_IPSEC_DIR_INBOUND))
+               return;
+
+       sess = get_sec_session_private_data(cop->sym->sec_session);
+       sa = &sess->sa;
+
+       mbuf = cop->sym->m_src;
+
+       switch (uc_compcode) {
+       case ROC_IE_OT_UCC_SUCCESS:
+               if (sa->ip_csum_enable)
+                       mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
+               break;
+       case ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM:
+               mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD;
+               break;
+       case ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM:
+               mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
+               if (sa->ip_csum_enable)
+                       mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
+               break;
+       case ROC_IE_OT_UCC_SUCCESS_PKT_L4_BADCSUM:
+               mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
+               if (sa->ip_csum_enable)
+                       mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
+               break;
+       default:
+               break;
+       }
+}
+
 static inline void
 cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
                               struct rte_crypto_op *cop,
@@ -357,17 +408,8 @@ cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
        if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC &&
            cop->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
                if (likely(compcode == CPT_COMP_WARN)) {
-                       if (unlikely(uc_compcode != ROC_IE_OT_UCC_SUCCESS)) {
-                               /* Success with additional info */
-                               switch (uc_compcode) {
-                               case ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST:
-                                       cop->aux_flags =
-                                               RTE_CRYPTO_OP_AUX_FLAGS_IPSEC_SOFT_EXPIRY;
-                                       break;
-                               default:
-                                       break;
-                               }
-                       }
+                       /* Success with additional info */
+                       cn10k_cpt_sec_ucc_process(cop, infl_req, uc_compcode);
                        cn10k_cpt_sec_post_process(cop, res);
                } else {
                        cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
@@ -475,18 +517,23 @@ cn10k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
        struct cnxk_cpt_qp *qp = qptr;
        struct pending_queue *pend_q;
        struct cpt_cn10k_res_s *res;
+       uint64_t infl_cnt, pq_tail;
        struct rte_crypto_op *cop;
-       int i, nb_pending;
+       int i;
 
        pend_q = &qp->pend_q;
 
-       nb_pending = pend_q->pending_count;
+       const uint64_t pq_mask = pend_q->pq_mask;
+
+       pq_tail = pend_q->tail;
+       infl_cnt = pending_queue_infl_cnt(pend_q->head, pq_tail, pq_mask);
+       nb_ops = RTE_MIN(nb_ops, infl_cnt);
 
-       if (nb_ops > nb_pending)
-               nb_ops = nb_pending;
+       /* Ensure infl_cnt isn't read before data lands */
+       rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
 
        for (i = 0; i < nb_ops; i++) {
-               infl_req = &pend_q->req_queue[pend_q->deq_head];
+               infl_req = &pend_q->req_queue[pq_tail];
 
                res = (struct cpt_cn10k_res_s *)&infl_req->res;
 
@@ -501,7 +548,7 @@ cn10k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
                        break;
                }
 
-               MOD_INC(pend_q->deq_head, qp->lf.nb_desc);
+               pending_queue_advance(&pq_tail, pq_mask);
 
                cop = infl_req->cop;
 
@@ -513,7 +560,7 @@ cn10k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
                        rte_mempool_put(qp->meta_info.pool, infl_req->mdata);
        }
 
-       pend_q->pending_count -= i;
+       pend_q->tail = pq_tail;
 
        return i;
 }