crypto/cnxk: add CPT hardware flow control checks

Add hardware supported flow control checks before enqueueing to CPT.
Since both poll mode and event mode can be used at the same time, add
hardware flow control checks to make sure s/w doesn't over submit to
hardware queues. For cn9k, queue depth usage is not high and so FC check
is omitted for poll mode.

To allow for more accurate updates, flow control hardware setting is
updated to give an update per 32 packets. In case of crypto adapter,
multiple cores can enqueue to the same CPT LF at the same time. To
allow such a case, flow control threshold is updated when the adapter
is configured.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>

diff --git a/drivers/common/cnxk/hw/cpt.h b/drivers/common/cnxk/hw/cpt.h
index 8fbba2c2a4da88cacb203604dc2b0b4ab1233063..3c87a0d1e4b271c2c962192b038f9f09e53e590c 100644 (file)
--- a/drivers/common/cnxk/hw/cpt.h
+++ b/drivers/common/cnxk/hw/cpt.h
@@ -322,4 +322,13 @@ struct cpt_frag_info_s {
        } w1;
 };
 
+union cpt_fc_write_s {
+       struct {
+               uint32_t qsize;
+               uint32_t reserved_32_63;
+               uint64_t reserved_64_127;
+       } s;
+       uint64_t u64[2];
+};
+
 #endif /* __CPT_HW_H__ */

diff --git a/drivers/common/cnxk/roc_cpt.c b/drivers/common/cnxk/roc_cpt.c
index e5b179e8e1173f70e3ce2f99fc3ab5bc934583b3..f1be6a34012063ec062e931488fc66cc8cc5017a 100644 (file)
--- a/drivers/common/cnxk/roc_cpt.c
+++ b/drivers/common/cnxk/roc_cpt.c
@@ -21,8 +21,9 @@
 #define CPT_IQ_GRP_SIZE(nb_desc)                                               \
        (CPT_IQ_NB_DESC_SIZE_DIV40(nb_desc) * CPT_IQ_GRP_LEN)
 
-#define CPT_LF_MAX_NB_DESC     128000
-#define CPT_LF_DEFAULT_NB_DESC 1024
+#define CPT_LF_MAX_NB_DESC     128000
+#define CPT_LF_DEFAULT_NB_DESC 1024
+#define CPT_LF_FC_MIN_THRESHOLD 32
 
 static void
 cpt_lf_misc_intr_enb_dis(struct roc_cpt_lf *lf, bool enb)
@@ -474,8 +475,6 @@ cpt_iq_init(struct roc_cpt_lf *lf)
        plt_write64(lf_q_size.u, lf->rbase + CPT_LF_Q_SIZE);
 
        lf->fc_addr = (uint64_t *)addr;
-       lf->fc_hyst_bits = plt_log2_u32(lf->nb_desc) / 2;
-       lf->fc_thresh = lf->nb_desc - (lf->nb_desc % (1 << lf->fc_hyst_bits));
 }
 
 int
@@ -879,7 +878,7 @@ roc_cpt_iq_enable(struct roc_cpt_lf *lf)
        lf_ctl.s.ena = 1;
        lf_ctl.s.fc_ena = 1;
        lf_ctl.s.fc_up_crossing = 0;
-       lf_ctl.s.fc_hyst_bits = lf->fc_hyst_bits;
+       lf_ctl.s.fc_hyst_bits = plt_log2_u32(CPT_LF_FC_MIN_THRESHOLD);
        plt_write64(lf_ctl.u, lf->rbase + CPT_LF_CTL);
 
        /* Enable command queue execution */
@@ -906,6 +905,7 @@ roc_cpt_lmtline_init(struct roc_cpt *roc_cpt, struct roc_cpt_lmtline *lmtline,
 
        lmtline->fc_addr = lf->fc_addr;
        lmtline->lmt_base = lf->lmt_base;
+       lmtline->fc_thresh = lf->nb_desc - CPT_LF_FC_MIN_THRESHOLD;
 
        return 0;
 }

diff --git a/drivers/common/cnxk/roc_cpt.h b/drivers/common/cnxk/roc_cpt.h
index 1b2032b547769e2c2ac7d2df48f77e374c11b082..a3a65f1e94d561ff5a706125cf5dbf23141492bf 100644 (file)
--- a/drivers/common/cnxk/roc_cpt.h
+++ b/drivers/common/cnxk/roc_cpt.h
@@ -99,6 +99,7 @@ struct roc_cpt_lmtline {
        uint64_t io_addr;
        uint64_t *fc_addr;
        uintptr_t lmt_base;
+       uint32_t fc_thresh;
 };
 
 struct roc_cpt_lf {
@@ -114,8 +115,6 @@ struct roc_cpt_lf {
        uint16_t msixoff;
        uint16_t pf_func;
        uint64_t *fc_addr;
-       uint32_t fc_hyst_bits;
-       uint64_t fc_thresh;
        uint64_t io_addr;
        uint8_t *iq_vaddr;
        struct roc_nix *inl_outb_nix;
@@ -144,15 +143,6 @@ struct roc_cpt_rxc_time_cfg {
        uint16_t zombie_thres;
 };
 
-static inline int
-roc_cpt_is_iq_full(struct roc_cpt_lf *lf)
-{
-       if (*lf->fc_addr < lf->fc_thresh)
-               return 0;
-
-       return 1;
-}
-
 int __roc_api roc_cpt_rxc_time_cfg(struct roc_cpt *roc_cpt,
                                   struct roc_cpt_rxc_time_cfg *cfg);
 int __roc_api roc_cpt_dev_init(struct roc_cpt *roc_cpt);

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 869fde01761e67fbcc50dff32d80b582f1c69913..f761ba36e240f85bb6cb3edacffadd165a08fdd3 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -194,6 +194,8 @@ cn10k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
        struct cnxk_cpt_qp *qp = qptr;
        struct pending_queue *pend_q;
        struct cpt_inst_s *inst;
+       union cpt_fc_write_s fc;
+       uint64_t *fc_addr;
        uint16_t lmt_id;
        uint64_t head;
        int ret, i;
@@ -211,11 +213,20 @@ cn10k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 
        lmt_base = qp->lmtline.lmt_base;
        io_addr = qp->lmtline.io_addr;
+       fc_addr = qp->lmtline.fc_addr;
+
+       const uint32_t fc_thresh = qp->lmtline.fc_thresh;
 
        ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
        inst = (struct cpt_inst_s *)lmt_base;
 
 again:
+       fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
+       if (unlikely(fc.s.qsize > fc_thresh)) {
+               i = 0;
+               goto pend_q_commit;
+       }
+
        for (i = 0; i < RTE_MIN(PKTS_PER_LOOP, nb_ops); i++) {
                infl_req = &pend_q->req_queue[head];
                infl_req->op_flags = 0;
@@ -386,7 +397,9 @@ cn10k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
        struct cpt_inflight_req *infl_req;
        uint64_t lmt_base, lmt_arg, w2;
        struct cpt_inst_s *inst;
+       union cpt_fc_write_s fc;
        struct cnxk_cpt_qp *qp;
+       uint64_t *fc_addr;
        uint16_t lmt_id;
        int ret;
 
@@ -408,6 +421,10 @@ cn10k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
        infl_req->op_flags = 0;
 
        lmt_base = qp->lmtline.lmt_base;
+       fc_addr = qp->lmtline.fc_addr;
+
+       const uint32_t fc_thresh = qp->lmtline.fc_thresh;
+
        ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
        inst = (struct cpt_inst_s *)lmt_base;
 
@@ -426,7 +443,8 @@ cn10k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
        inst->w2.u64 = w2;
        inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
 
-       if (roc_cpt_is_iq_full(&qp->lf)) {
+       fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
+       if (unlikely(fc.s.qsize > fc_thresh)) {
                rte_mempool_put(qp->ca.req_mp, infl_req);
                rte_errno = EAGAIN;
                return 0;

diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 06dc18d195691ac937b9803af1d603ec9e0409d2..3c4cc41bdb4c8a141577ed0de46af48cdd0d3a7a 100644 (file)
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -452,8 +452,10 @@ uint16_t
 cn9k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
 {
        struct cpt_inflight_req *infl_req;
+       union cpt_fc_write_s fc;
        struct cnxk_cpt_qp *qp;
        struct cpt_inst_s inst;
+       uint64_t *fc_addr;
        int ret;
 
        ret = cn9k_ca_meta_info_extract(op, &qp, &inst);
@@ -487,7 +489,12 @@ cn9k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
        inst.res_addr = (uint64_t)&infl_req->res;
        inst.w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
 
-       if (roc_cpt_is_iq_full(&qp->lf)) {
+       fc_addr = qp->lmtline.fc_addr;
+
+       const uint32_t fc_thresh = qp->lmtline.fc_thresh;
+
+       fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
+       if (unlikely(fc.s.qsize > fc_thresh)) {
                rte_mempool_put(qp->ca.req_mp, infl_req);
                rte_errno = EAGAIN;
                return 0;

diff --git a/drivers/event/cnxk/cnxk_eventdev.c b/drivers/event/cnxk/cnxk_eventdev.c
index b66f241ef87dc41b7e996b84cd1a871c25698905..a9e4201ed89e2e2369fec1ed38c33296f18bce8e 100644 (file)
--- a/drivers/event/cnxk/cnxk_eventdev.c
+++ b/drivers/event/cnxk/cnxk_eventdev.c
@@ -12,6 +12,24 @@ crypto_adapter_qp_setup(const struct rte_cryptodev *cdev,
        char name[RTE_MEMPOOL_NAMESIZE];
        uint32_t cache_size, nb_req;
        unsigned int req_size;
+       uint32_t nb_desc_min;
+
+       /*
+        * Update CPT FC threshold. Decrement by hardware burst size to allow
+        * simultaneous enqueue from all available cores.
+        */
+       if (roc_model_is_cn10k())
+               nb_desc_min = rte_lcore_count() * 32;
+       else
+               nb_desc_min = rte_lcore_count() * 2;
+
+       if (qp->lmtline.fc_thresh < nb_desc_min) {
+               plt_err("CPT queue depth not sufficient to allow enqueueing from %d cores",
+                       rte_lcore_count());
+               return -ENOSPC;
+       }
+
+       qp->lmtline.fc_thresh -= nb_desc_min;
 
        snprintf(name, RTE_MEMPOOL_NAMESIZE, "cnxk_ca_req_%u:%u",
                 cdev->data->dev_id, qp->lf.lf_id);
@@ -69,9 +87,18 @@ cnxk_crypto_adapter_qp_add(const struct rte_eventdev *event_dev,
 static int
 crypto_adapter_qp_free(struct cnxk_cpt_qp *qp)
 {
+       int ret;
+
        rte_mempool_free(qp->ca.req_mp);
        qp->ca.enabled = false;
 
+       ret = roc_cpt_lmtline_init(qp->lf.roc_cpt, &qp->lmtline, qp->lf.lf_id);
+       if (ret < 0) {
+               plt_err("Could not reset lmtline for queue pair %d",
+                       qp->lf.lf_id);
+               return ret;
+       }
+
        return 0;
 }