crypto/cnxk: add dequeue burst

Add dequeue_burst op in cn9k & cn10k.

Signed-off-by: Ankur Dwivedi <adwivedi@marvell.com>
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Archana Muniganti <marchana@marvell.com>
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>

diff --git a/doc/guides/cryptodevs/features/cn10k.ini b/doc/guides/cryptodevs/features/cn10k.ini
index 0aa097d..7f433fa 100644 (file)
--- a/doc/guides/cryptodevs/features/cn10k.ini
+++ b/doc/guides/cryptodevs/features/cn10k.ini
@@ -4,6 +4,9 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
+HW Accelerated         = Y
+Symmetric sessionless  = Y
 
 ;
 ; Supported crypto algorithms of 'cn10k' crypto driver.

diff --git a/doc/guides/cryptodevs/features/cn9k.ini b/doc/guides/cryptodevs/features/cn9k.ini
index 64ee929..9c9d54d 100644 (file)
--- a/doc/guides/cryptodevs/features/cn9k.ini
+++ b/doc/guides/cryptodevs/features/cn9k.ini
@@ -4,6 +4,9 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
+HW Accelerated         = Y
+Symmetric sessionless  = Y
 
 ;
 ; Supported crypto algorithms of 'cn9k' crypto driver.

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev.c b/drivers/crypto/cnxk/cn10k_cryptodev.c
index 53f7a94..31addc0 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_cryptodev.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev.c
@@ -80,6 +80,10 @@ cn10k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
        dev->dev_ops = &cn10k_cpt_ops;
        dev->driver_id = cn10k_cryptodev_driver_id;
 
+       dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+                            RTE_CRYPTODEV_FF_HW_ACCELERATED |
+                            RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+
        cn10k_cpt_set_enqdeq_fns(dev);
 
        return 0;

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 5dd2cd2..ec301f4 100644 (file)
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -190,10 +190,115 @@ update_pending:
        return count + i;
 }
 
+static inline void
+cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
+                              struct rte_crypto_op *cop,
+                              struct cpt_inflight_req *infl_req)
+{
+       struct cpt_cn10k_res_s *res = (struct cpt_cn10k_res_s *)&infl_req->res;
+       unsigned int sz;
+
+       if (likely(res->compcode == CPT_COMP_GOOD ||
+                  res->compcode == CPT_COMP_WARN)) {
+               if (unlikely(res->uc_compcode)) {
+                       cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+
+                       plt_dp_info("Request failed with microcode error");
+                       plt_dp_info("MC completion code 0x%x",
+                                   res->uc_compcode);
+                       goto temp_sess_free;
+               }
+
+               cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+       } else {
+               cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+               plt_dp_info("HW completion code 0x%x", res->compcode);
+
+               switch (res->compcode) {
+               case CPT_COMP_INSTERR:
+                       plt_dp_err("Request failed with instruction error");
+                       break;
+               case CPT_COMP_FAULT:
+                       plt_dp_err("Request failed with DMA fault");
+                       break;
+               case CPT_COMP_HWERR:
+                       plt_dp_err("Request failed with hardware error");
+                       break;
+               default:
+                       plt_dp_err(
+                               "Request failed with unknown completion code");
+               }
+       }
+
+temp_sess_free:
+       if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+               if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+                       sym_session_clear(cn10k_cryptodev_driver_id,
+                                         cop->sym->session);
+                       sz = rte_cryptodev_sym_get_existing_header_session_size(
+                               cop->sym->session);
+                       memset(cop->sym->session, 0, sz);
+                       rte_mempool_put(qp->sess_mp, cop->sym->session);
+                       cop->sym->session = NULL;
+               }
+       }
+}
+
+static uint16_t
+cn10k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+       struct cpt_inflight_req *infl_req;
+       struct cnxk_cpt_qp *qp = qptr;
+       struct pending_queue *pend_q;
+       struct cpt_cn10k_res_s *res;
+       struct rte_crypto_op *cop;
+       int i, nb_pending;
+
+       pend_q = &qp->pend_q;
+
+       nb_pending = pend_q->pending_count;
+
+       if (nb_ops > nb_pending)
+               nb_ops = nb_pending;
+
+       for (i = 0; i < nb_ops; i++) {
+               infl_req = &pend_q->req_queue[pend_q->deq_head];
+
+               res = (struct cpt_cn10k_res_s *)&infl_req->res;
+
+               if (unlikely(res->compcode == CPT_COMP_NOT_DONE)) {
+                       if (unlikely(rte_get_timer_cycles() >
+                                    pend_q->time_out)) {
+                               plt_err("Request timed out");
+                               pend_q->time_out = rte_get_timer_cycles() +
+                                                  DEFAULT_COMMAND_TIMEOUT *
+                                                          rte_get_timer_hz();
+                       }
+                       break;
+               }
+
+               MOD_INC(pend_q->deq_head, qp->lf.nb_desc);
+
+               cop = infl_req->cop;
+
+               ops[i] = cop;
+
+               cn10k_cpt_dequeue_post_process(qp, cop, infl_req);
+
+               if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF))
+                       rte_mempool_put(qp->meta_info.pool, infl_req->mdata);
+       }
+
+       pend_q->pending_count -= i;
+
+       return i;
+}
+
 void
 cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev)
 {
        dev->enqueue_burst = cn10k_cpt_enqueue_burst;
+       dev->dequeue_burst = cn10k_cpt_dequeue_burst;
 
        rte_mb();
 }

diff --git a/drivers/crypto/cnxk/cn9k_cryptodev.c b/drivers/crypto/cnxk/cn9k_cryptodev.c
index 4dbb40d..7908896 100644 (file)
--- a/drivers/crypto/cnxk/cn9k_cryptodev.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev.c
@@ -78,6 +78,10 @@ cn9k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
        dev->dev_ops = &cn9k_cpt_ops;
        dev->driver_id = cn9k_cryptodev_driver_id;
 
+       dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+                            RTE_CRYPTODEV_FF_HW_ACCELERATED |
+                            RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+
        cn9k_cpt_set_enqdeq_fns(dev);
 
        return 0;

diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index f09f9ee..27076a8 100644 (file)
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -155,10 +155,113 @@ cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
        return count;
 }
 
+static inline void
+cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop,
+                             struct cpt_inflight_req *infl_req)
+{
+       struct cpt_cn9k_res_s *res = (struct cpt_cn9k_res_s *)&infl_req->res;
+       unsigned int sz;
+
+       if (likely(res->compcode == CPT_COMP_GOOD)) {
+               if (unlikely(res->uc_compcode)) {
+                       cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+
+                       plt_dp_info("Request failed with microcode error");
+                       plt_dp_info("MC completion code 0x%x",
+                                   res->uc_compcode);
+                       goto temp_sess_free;
+               }
+
+               cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+       } else {
+               cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+               plt_dp_info("HW completion code 0x%x", res->compcode);
+
+               switch (res->compcode) {
+               case CPT_COMP_INSTERR:
+                       plt_dp_err("Request failed with instruction error");
+                       break;
+               case CPT_COMP_FAULT:
+                       plt_dp_err("Request failed with DMA fault");
+                       break;
+               case CPT_COMP_HWERR:
+                       plt_dp_err("Request failed with hardware error");
+                       break;
+               default:
+                       plt_dp_err(
+                               "Request failed with unknown completion code");
+               }
+       }
+
+temp_sess_free:
+       if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+               if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+                       sym_session_clear(cn9k_cryptodev_driver_id,
+                                         cop->sym->session);
+                       sz = rte_cryptodev_sym_get_existing_header_session_size(
+                               cop->sym->session);
+                       memset(cop->sym->session, 0, sz);
+                       rte_mempool_put(qp->sess_mp, cop->sym->session);
+                       cop->sym->session = NULL;
+               }
+       }
+}
+
+static uint16_t
+cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+       struct cnxk_cpt_qp *qp = qptr;
+       struct pending_queue *pend_q;
+       struct cpt_inflight_req *infl_req;
+       struct cpt_cn9k_res_s *res;
+       struct rte_crypto_op *cop;
+       uint32_t pq_deq_head;
+       int i;
+
+       pend_q = &qp->pend_q;
+
+       nb_ops = RTE_MIN(nb_ops, pend_q->pending_count);
+
+       pq_deq_head = pend_q->deq_head;
+
+       for (i = 0; i < nb_ops; i++) {
+               infl_req = &pend_q->req_queue[pq_deq_head];
+
+               res = (struct cpt_cn9k_res_s *)&infl_req->res;
+
+               if (unlikely(res->compcode == CPT_COMP_NOT_DONE)) {
+                       if (unlikely(rte_get_timer_cycles() >
+                                    pend_q->time_out)) {
+                               plt_err("Request timed out");
+                               pend_q->time_out = rte_get_timer_cycles() +
+                                                  DEFAULT_COMMAND_TIMEOUT *
+                                                          rte_get_timer_hz();
+                       }
+                       break;
+               }
+
+               MOD_INC(pq_deq_head, qp->lf.nb_desc);
+
+               cop = infl_req->cop;
+
+               ops[i] = cop;
+
+               cn9k_cpt_dequeue_post_process(qp, cop, infl_req);
+
+               if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF))
+                       rte_mempool_put(qp->meta_info.pool, infl_req->mdata);
+       }
+
+       pend_q->pending_count -= i;
+       pend_q->deq_head = pq_deq_head;
+
+       return i;
+}
 void
 cn9k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev)
 {
        dev->enqueue_burst = cn9k_cpt_enqueue_burst;
+       dev->dequeue_burst = cn9k_cpt_dequeue_burst;
 
        rte_mb();
 }