From 52048f8f89aff15b90d6dbf81ec665e5fc53d24d Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Wed, 25 Jul 2018 22:34:54 +0530 Subject: [PATCH] compress/octeontx: support burst enqueue/dequeue Implement enqueue/dequeue APIs to perform compression/decompression operations Signed-off-by: Ashish Gupta Signed-off-by: Shally Verma Signed-off-by: Sunila Sahu --- drivers/compress/octeontx/otx_zip.c | 47 ++++++++ drivers/compress/octeontx/otx_zip.h | 96 ++++++++++++++++- drivers/compress/octeontx/otx_zip_pmd.c | 138 ++++++++++++++++++++++++ 3 files changed, 280 insertions(+), 1 deletion(-) diff --git a/drivers/compress/octeontx/otx_zip.c b/drivers/compress/octeontx/otx_zip.c index 7c81db110b..a9046ff351 100644 --- a/drivers/compress/octeontx/otx_zip.c +++ b/drivers/compress/octeontx/otx_zip.c @@ -88,6 +88,53 @@ zipvf_q_term(struct zipvf_qp *qp) return 0; } +void +zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *cmd) +{ + zip_quex_doorbell_t dbell; + union zip_nptr_s ncp; + uint64_t *ncb_ptr; + struct zipvf_cmdq *cmdq = &qp->cmdq; + void *reg_base = qp->vf->vbar0; + + /*Held queue lock*/ + rte_spinlock_lock(&(cmdq->qlock)); + + /* Check space availability in zip cmd queue */ + if ((((cmdq->sw_head - (uint64_t *)cmdq->va) * sizeof(uint64_t *)) + + ZIP_CMD_SIZE) == (ZIP_MAX_CMDQ_SIZE - ZIP_MAX_NCBP_SIZE)) { + /*Last buffer of the command queue*/ + memcpy((uint8_t *)cmdq->sw_head, + (uint8_t *)cmd, + sizeof(union zip_inst_s)); + /* move pointer to next loc in unit of 64-bit word */ + cmdq->sw_head += ZIP_CMD_SIZE_WORDS; + + /* now, point the "Next-Chunk Buffer Ptr" to sw_head */ + ncb_ptr = cmdq->sw_head; + /* Pointing head again to cmdqueue base*/ + cmdq->sw_head = (uint64_t *)cmdq->va; + + ncp.u = 0ull; + ncp.s.addr = cmdq->iova; + *ncb_ptr = ncp.u; + } else { + /*Enough buffers available in the command queue*/ + memcpy((uint8_t *)cmdq->sw_head, + (uint8_t *)cmd, + sizeof(union zip_inst_s)); + cmdq->sw_head += ZIP_CMD_SIZE_WORDS; + } + + rte_wmb(); + + /* Ringing ZIP VF doorbell */ + dbell.u = 0ull; + dbell.s.dbell_cnt = 1; + zip_reg_write64(reg_base, ZIP_VQ_DOORBELL, dbell.u); + + rte_spinlock_unlock(&(cmdq->qlock)); +} int zipvf_create(struct rte_compressdev *compressdev) diff --git a/drivers/compress/octeontx/otx_zip.h b/drivers/compress/octeontx/otx_zip.h index 3fcd86a86f..99a38d005b 100644 --- a/drivers/compress/octeontx/otx_zip.h +++ b/drivers/compress/octeontx/otx_zip.h @@ -159,6 +159,95 @@ struct zip_vf { /* pointer to pools */ } __rte_cache_aligned; + +static inline void +zipvf_prepare_in_buf(struct zip_stream *zstrm, struct rte_comp_op *op) +{ + uint32_t offset, inlen; + struct rte_mbuf *m_src; + union zip_inst_s *inst = zstrm->inst; + + inlen = op->src.length; + offset = op->src.offset; + m_src = op->m_src; + + /* Prepare direct input data pointer */ + inst->s.dg = 0; + inst->s.inp_ptr_addr.s.addr = + rte_pktmbuf_iova_offset(m_src, offset); + inst->s.inp_ptr_ctl.s.length = inlen; +} + +static inline void +zipvf_prepare_out_buf(struct zip_stream *zstrm, struct rte_comp_op *op) +{ + uint32_t offset; + struct rte_mbuf *m_dst; + union zip_inst_s *inst = zstrm->inst; + + offset = op->dst.offset; + m_dst = op->m_dst; + + /* Prepare direct input data pointer */ + inst->s.ds = 0; + inst->s.out_ptr_addr.s.addr = + rte_pktmbuf_iova_offset(m_dst, offset); + inst->s.totaloutputlength = rte_pktmbuf_pkt_len(m_dst) - + op->dst.offset; + inst->s.out_ptr_ctl.s.length = inst->s.totaloutputlength; +} + +static inline void +zipvf_prepare_cmd_stateless(struct rte_comp_op *op, struct zip_stream *zstrm) +{ + union zip_inst_s *inst = zstrm->inst; + + /* set flush flag to always 1*/ + inst->s.ef = 1; + + if (inst->s.op == ZIP_OP_E_DECOMP) + inst->s.sf = 1; + else + inst->s.sf = 0; + + /* Set input checksum */ + inst->s.adlercrc32 = op->input_chksum; + + /* Prepare gather buffers */ + zipvf_prepare_in_buf(zstrm, op); + zipvf_prepare_out_buf(zstrm, op); +} + +#ifdef ZIP_DBG +static inline void +zip_dump_instruction(void *inst) +{ + union zip_inst_s *cmd83 = (union zip_inst_s *)inst; + printf("####### START ########\n"); + printf("doneint:%d totaloutputlength:%d\n", cmd83->s.doneint, + cmd83->s.totaloutputlength); + printf("exnum:%d iv:%d exbits:%d hmif:%d halg:%d\n", cmd83->s.exn, + cmd83->s.iv, cmd83->s.exbits, cmd83->s.hmif, cmd83->s.halg); + printf("flush:%d speed:%d cc:%d\n", cmd83->s.sf, + cmd83->s.ss, cmd83->s.cc); + printf("eof:%d bof:%d op:%d dscatter:%d dgather:%d hgather:%d\n", + cmd83->s.ef, cmd83->s.bf, cmd83->s.op, cmd83->s.ds, + cmd83->s.dg, cmd83->s.hg); + printf("historylength:%d adler32:%d\n", cmd83->s.historylength, + cmd83->s.adlercrc32); + printf("ctx_ptr.addr:0x%"PRIx64"\n", cmd83->s.ctx_ptr_addr.s.addr); + printf("ctx_ptr.len:%d\n", cmd83->s.ctx_ptr_ctl.s.length); + printf("history_ptr.addr:0x%"PRIx64"\n", cmd83->s.his_ptr_addr.s.addr); + printf("history_ptr.len:%d\n", cmd83->s.his_ptr_ctl.s.length); + printf("inp_ptr.addr:0x%"PRIx64"\n", cmd83->s.inp_ptr_addr.s.addr); + printf("inp_ptr.len:%d\n", cmd83->s.inp_ptr_ctl.s.length); + printf("out_ptr.addr:0x%"PRIx64"\n", cmd83->s.out_ptr_addr.s.addr); + printf("out_ptr.len:%d\n", cmd83->s.out_ptr_ctl.s.length); + printf("result_ptr.len:%d\n", cmd83->s.res_ptr_ctl.s.length); + printf("####### END ########\n"); +} +#endif + int zipvf_create(struct rte_compressdev *compressdev); @@ -171,9 +260,14 @@ zipvf_q_init(struct zipvf_qp *qp); int zipvf_q_term(struct zipvf_qp *qp); -int +void zipvf_push_command(struct zipvf_qp *qp, union zip_inst_s *zcmd); +int +zip_process_op(struct rte_comp_op *op, + struct zipvf_qp *qp, + struct zip_stream *zstrm); + uint64_t zip_reg_read64(uint8_t *hw_addr, uint64_t offset); diff --git a/drivers/compress/octeontx/otx_zip_pmd.c b/drivers/compress/octeontx/otx_zip_pmd.c index 16f581bd87..9d13f93319 100644 --- a/drivers/compress/octeontx/otx_zip_pmd.c +++ b/drivers/compress/octeontx/otx_zip_pmd.c @@ -28,6 +28,85 @@ static const struct rte_compressdev_capabilities RTE_COMP_END_OF_CAPABILITIES_LIST() }; +/* + * Reset session to default state for next set of stateless operation + */ +static inline void +reset_stream(struct zip_stream *z_stream) +{ + union zip_inst_s *inst = (union zip_inst_s *)(z_stream->inst); + + inst->s.bf = 1; + inst->s.ef = 0; +} + +int +zip_process_op(struct rte_comp_op *op, + struct zipvf_qp *qp, + struct zip_stream *zstrm) +{ + union zip_inst_s *inst = zstrm->inst; + volatile union zip_zres_s *zresult = NULL; + + + if ((op->m_src->nb_segs > 1) || (op->m_dst->nb_segs > 1) || + (op->src.offset > rte_pktmbuf_pkt_len(op->m_src)) || + (op->dst.offset > rte_pktmbuf_pkt_len(op->m_dst))) { + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + ZIP_PMD_ERR("Segmented packet is not supported\n"); + return 0; + } + + zipvf_prepare_cmd_stateless(op, zstrm); + + zresult = (union zip_zres_s *)zstrm->bufs[RES_BUF]; + zresult->s.compcode = 0; + +#ifdef ZIP_DBG + zip_dump_instruction(inst); +#endif + + /* Submit zip command */ + zipvf_push_command(qp, (void *)inst); + + /* Check and Process results in sync mode */ + do { + } while (!zresult->s.compcode); + + if (zresult->s.compcode == ZIP_COMP_E_SUCCESS) { + op->status = RTE_COMP_OP_STATUS_SUCCESS; + } else { + /* FATAL error cannot do anything */ + ZIP_PMD_ERR("operation failed with error code:%d\n", + zresult->s.compcode); + if (zresult->s.compcode == ZIP_COMP_E_DSTOP) + op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + else + op->status = RTE_COMP_OP_STATUS_ERROR; + } + + ZIP_PMD_INFO("written %d\n", zresult->s.totalbyteswritten); + + /* Update op stats */ + switch (op->status) { + case RTE_COMP_OP_STATUS_SUCCESS: + op->consumed = zresult->s.totalbytesread; + /* Fall-through */ + case RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED: + op->produced = zresult->s.totalbyteswritten; + break; + default: + ZIP_PMD_ERR("stats not updated for status:%d\n", + op->status); + break; + } + /* zstream is reset irrespective of result */ + reset_stream(zstrm); + + zresult->s.compcode = ZIP_COMP_E_NOTDONE; + return 0; +} + /** Parse xform parameters and setup a stream */ static int zip_set_stream_parameters(struct rte_compressdev *dev, @@ -114,6 +193,7 @@ zip_set_stream_parameters(struct rte_compressdev *dev, inst->s.res_ptr_ctl.s.length = 0; z_stream->inst = inst; + z_stream->func = zip_process_op; return 0; @@ -397,6 +477,62 @@ zip_pmd_stream_free(struct rte_compressdev *dev, void *stream) } +static uint16_t +zip_pmd_enqueue_burst_sync(void *queue_pair, + struct rte_comp_op **ops, uint16_t nb_ops) +{ + struct zipvf_qp *qp = queue_pair; + struct rte_comp_op *op; + struct zip_stream *zstrm; + int i, ret = 0; + uint16_t enqd = 0; + + for (i = 0; i < nb_ops; i++) { + op = ops[i]; + + if (op->op_type == RTE_COMP_OP_STATEFUL) { + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + } else { + /* process stateless ops */ + zstrm = (struct zip_stream *)op->private_xform; + if (unlikely(zstrm == NULL)) + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + else + ret = zstrm->func(op, qp, zstrm); + } + + /* Whatever is out of op, put it into completion queue with + * its status + */ + if (!ret) + ret = rte_ring_enqueue(qp->processed_pkts, (void *)op); + + if (unlikely(ret < 0)) { + /* increment count if failed to enqueue op*/ + qp->qp_stats.enqueue_err_count++; + } else { + qp->qp_stats.enqueued_count++; + enqd++; + } + } + return enqd; +} + +static uint16_t +zip_pmd_dequeue_burst_sync(void *queue_pair, + struct rte_comp_op **ops, uint16_t nb_ops) +{ + struct zipvf_qp *qp = queue_pair; + + unsigned int nb_dequeued = 0; + + nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, + (void **)ops, nb_ops, NULL); + qp->qp_stats.dequeued_count += nb_dequeued; + + return nb_dequeued; +} + struct rte_compressdev_ops octtx_zip_pmd_ops = { .dev_configure = zip_pmd_config, .dev_start = zip_pmd_start, @@ -458,6 +594,8 @@ zip_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, compressdev->dev_ops = &octtx_zip_pmd_ops; /* register rx/tx burst functions for data path */ + compressdev->dequeue_burst = zip_pmd_dequeue_burst_sync; + compressdev->enqueue_burst = zip_pmd_enqueue_burst_sync; compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED; return ret; } -- 2.20.1