+static inline int
+acc100_dma_desc_te_fill(struct rte_bbdev_enc_op *op,
+ struct acc100_dma_req_desc *desc, struct rte_mbuf **input,
+ struct rte_mbuf *output, uint32_t *in_offset,
+ uint32_t *out_offset, uint32_t *out_length,
+ uint32_t *mbuf_total_left, uint32_t *seg_total_left, uint8_t r)
+{
+ int next_triplet = 1; /* FCW already done */
+ uint32_t e, ea, eb, length;
+ uint16_t k, k_neg, k_pos;
+ uint8_t cab, c_neg;
+
+ desc->word0 = ACC100_DMA_DESC_TYPE;
+ desc->word1 = 0; /**< Timestamp could be disabled */
+ desc->word2 = 0;
+ desc->word3 = 0;
+ desc->numCBs = 1;
+
+ if (op->turbo_enc.code_block_mode == 0) {
+ ea = op->turbo_enc.tb_params.ea;
+ eb = op->turbo_enc.tb_params.eb;
+ cab = op->turbo_enc.tb_params.cab;
+ k_neg = op->turbo_enc.tb_params.k_neg;
+ k_pos = op->turbo_enc.tb_params.k_pos;
+ c_neg = op->turbo_enc.tb_params.c_neg;
+ e = (r < cab) ? ea : eb;
+ k = (r < c_neg) ? k_neg : k_pos;
+ } else {
+ e = op->turbo_enc.cb_params.e;
+ k = op->turbo_enc.cb_params.k;
+ }
+
+ if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH))
+ length = (k - 24) >> 3;
+ else
+ length = k >> 3;
+
+ if (unlikely((*mbuf_total_left == 0) || (*mbuf_total_left < length))) {
+ rte_bbdev_log(ERR,
+ "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
+ *mbuf_total_left, length);
+ return -1;
+ }
+
+ next_triplet = acc100_dma_fill_blk_type_in(desc, input, in_offset,
+ length, seg_total_left, next_triplet);
+ if (unlikely(next_triplet < 0)) {
+ rte_bbdev_log(ERR,
+ "Mismatch between data to process and mbuf data length in bbdev_op: %p",
+ op);
+ return -1;
+ }
+ desc->data_ptrs[next_triplet - 1].last = 1;
+ desc->m2dlen = next_triplet;
+ *mbuf_total_left -= length;
+
+ /* Set output length */
+ if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_RATE_MATCH))
+ /* Integer round up division by 8 */
+ *out_length = (e + 7) >> 3;
+ else
+ *out_length = (k >> 3) * 3 + 2;
+
+ next_triplet = acc100_dma_fill_blk_type_out(desc, output, *out_offset,
+ *out_length, next_triplet, ACC100_DMA_BLKID_OUT_ENC);
+ if (unlikely(next_triplet < 0)) {
+ rte_bbdev_log(ERR,
+ "Mismatch between data to process and mbuf data length in bbdev_op: %p",
+ op);
+ return -1;
+ }
+ op->turbo_enc.output.length += *out_length;
+ *out_offset += *out_length;
+ desc->data_ptrs[next_triplet - 1].last = 1;
+ desc->d2mlen = next_triplet - desc->m2dlen;
+
+ desc->op_addr = op;
+
+ return 0;
+}
+
+static inline int
+acc100_dma_desc_le_fill(struct rte_bbdev_enc_op *op,
+ struct acc100_dma_req_desc *desc, struct rte_mbuf **input,
+ struct rte_mbuf *output, uint32_t *in_offset,
+ uint32_t *out_offset, uint32_t *out_length,
+ uint32_t *mbuf_total_left, uint32_t *seg_total_left)
+{
+ int next_triplet = 1; /* FCW already done */
+ uint16_t K, in_length_in_bits, in_length_in_bytes;
+ struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc;
+
+ acc100_header_init(desc);
+
+ K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c;
+ in_length_in_bits = K - enc->n_filler;
+ if ((enc->op_flags & RTE_BBDEV_LDPC_CRC_24A_ATTACH) ||
+ (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH))
+ in_length_in_bits -= 24;
+ in_length_in_bytes = in_length_in_bits >> 3;
+
+ if (unlikely((*mbuf_total_left == 0) ||
+ (*mbuf_total_left < in_length_in_bytes))) {
+ rte_bbdev_log(ERR,
+ "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
+ *mbuf_total_left, in_length_in_bytes);
+ return -1;
+ }
+
+ next_triplet = acc100_dma_fill_blk_type_in(desc, input, in_offset,
+ in_length_in_bytes,
+ seg_total_left, next_triplet);
+ if (unlikely(next_triplet < 0)) {
+ rte_bbdev_log(ERR,
+ "Mismatch between data to process and mbuf data length in bbdev_op: %p",
+ op);
+ return -1;
+ }
+ desc->data_ptrs[next_triplet - 1].last = 1;
+ desc->m2dlen = next_triplet;
+ *mbuf_total_left -= in_length_in_bytes;
+
+ /* Set output length */
+ /* Integer round up division by 8 */
+ *out_length = (enc->cb_params.e + 7) >> 3;
+
+ next_triplet = acc100_dma_fill_blk_type_out(desc, output, *out_offset,
+ *out_length, next_triplet, ACC100_DMA_BLKID_OUT_ENC);
+ op->ldpc_enc.output.length += *out_length;
+ *out_offset += *out_length;
+ desc->data_ptrs[next_triplet - 1].last = 1;
+ desc->data_ptrs[next_triplet - 1].dma_ext = 0;
+ desc->d2mlen = next_triplet - desc->m2dlen;
+
+ desc->op_addr = op;
+
+ return 0;
+}
+
+static inline int
+acc100_dma_desc_td_fill(struct rte_bbdev_dec_op *op,
+ struct acc100_dma_req_desc *desc, struct rte_mbuf **input,
+ struct rte_mbuf *h_output, struct rte_mbuf *s_output,
+ uint32_t *in_offset, uint32_t *h_out_offset,
+ uint32_t *s_out_offset, uint32_t *h_out_length,
+ uint32_t *s_out_length, uint32_t *mbuf_total_left,
+ uint32_t *seg_total_left, uint8_t r)
+{
+ int next_triplet = 1; /* FCW already done */
+ uint16_t k;
+ uint16_t crc24_overlap = 0;
+ uint32_t e, kw;
+
+ desc->word0 = ACC100_DMA_DESC_TYPE;
+ desc->word1 = 0; /**< Timestamp could be disabled */
+ desc->word2 = 0;
+ desc->word3 = 0;
+ desc->numCBs = 1;
+
+ if (op->turbo_dec.code_block_mode == 0) {
+ k = (r < op->turbo_dec.tb_params.c_neg)
+ ? op->turbo_dec.tb_params.k_neg
+ : op->turbo_dec.tb_params.k_pos;
+ e = (r < op->turbo_dec.tb_params.cab)
+ ? op->turbo_dec.tb_params.ea
+ : op->turbo_dec.tb_params.eb;
+ } else {
+ k = op->turbo_dec.cb_params.k;
+ e = op->turbo_dec.cb_params.e;
+ }
+
+ if ((op->turbo_dec.code_block_mode == 0)
+ && !check_bit(op->turbo_dec.op_flags,
+ RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP))
+ crc24_overlap = 24;
+
+ /* Calculates circular buffer size.
+ * According to 3gpp 36.212 section 5.1.4.2
+ * Kw = 3 * Kpi,
+ * where:
+ * Kpi = nCol * nRow
+ * where nCol is 32 and nRow can be calculated from:
+ * D =< nCol * nRow
+ * where D is the size of each output from turbo encoder block (k + 4).
+ */
+ kw = RTE_ALIGN_CEIL(k + 4, 32) * 3;
+
+ if (unlikely((*mbuf_total_left == 0) || (*mbuf_total_left < kw))) {
+ rte_bbdev_log(ERR,
+ "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
+ *mbuf_total_left, kw);
+ return -1;
+ }
+
+ next_triplet = acc100_dma_fill_blk_type_in(desc, input, in_offset, kw,
+ seg_total_left, next_triplet);
+ if (unlikely(next_triplet < 0)) {
+ rte_bbdev_log(ERR,
+ "Mismatch between data to process and mbuf data length in bbdev_op: %p",
+ op);
+ return -1;
+ }
+ desc->data_ptrs[next_triplet - 1].last = 1;
+ desc->m2dlen = next_triplet;
+ *mbuf_total_left -= kw;
+
+ next_triplet = acc100_dma_fill_blk_type_out(
+ desc, h_output, *h_out_offset,
+ k >> 3, next_triplet, ACC100_DMA_BLKID_OUT_HARD);
+ if (unlikely(next_triplet < 0)) {
+ rte_bbdev_log(ERR,
+ "Mismatch between data to process and mbuf data length in bbdev_op: %p",
+ op);
+ return -1;
+ }
+
+ *h_out_length = ((k - crc24_overlap) >> 3);
+ op->turbo_dec.hard_output.length += *h_out_length;
+ *h_out_offset += *h_out_length;
+
+ /* Soft output */
+ if (check_bit(op->turbo_dec.op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
+ if (check_bit(op->turbo_dec.op_flags,
+ RTE_BBDEV_TURBO_EQUALIZER))
+ *s_out_length = e;
+ else
+ *s_out_length = (k * 3) + 12;
+
+ next_triplet = acc100_dma_fill_blk_type_out(desc, s_output,
+ *s_out_offset, *s_out_length, next_triplet,
+ ACC100_DMA_BLKID_OUT_SOFT);
+ if (unlikely(next_triplet < 0)) {
+ rte_bbdev_log(ERR,
+ "Mismatch between data to process and mbuf data length in bbdev_op: %p",
+ op);
+ return -1;
+ }
+
+ op->turbo_dec.soft_output.length += *s_out_length;
+ *s_out_offset += *s_out_length;
+ }
+
+ desc->data_ptrs[next_triplet - 1].last = 1;
+ desc->d2mlen = next_triplet - desc->m2dlen;
+
+ desc->op_addr = op;
+
+ return 0;
+}
+
+static inline int
+acc100_dma_desc_ld_fill(struct rte_bbdev_dec_op *op,
+ struct acc100_dma_req_desc *desc,
+ struct rte_mbuf **input, struct rte_mbuf *h_output,
+ uint32_t *in_offset, uint32_t *h_out_offset,
+ uint32_t *h_out_length, uint32_t *mbuf_total_left,
+ uint32_t *seg_total_left,
+ struct acc100_fcw_ld *fcw)
+{
+ struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec;
+ int next_triplet = 1; /* FCW already done */
+ uint32_t input_length;
+ uint16_t output_length, crc24_overlap = 0;
+ uint16_t sys_cols, K, h_p_size, h_np_size;
+ bool h_comp = check_bit(dec->op_flags,
+ RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION);
+
+ acc100_header_init(desc);
+
+ if (check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP))
+ crc24_overlap = 24;
+
+ /* Compute some LDPC BG lengths */
+ input_length = dec->cb_params.e;
+ if (check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_LLR_COMPRESSION))
+ input_length = (input_length * 3 + 3) / 4;
+ sys_cols = (dec->basegraph == 1) ? 22 : 10;
+ K = sys_cols * dec->z_c;
+ output_length = K - dec->n_filler - crc24_overlap;
+
+ if (unlikely((*mbuf_total_left == 0) ||
+ (*mbuf_total_left < input_length))) {
+ rte_bbdev_log(ERR,
+ "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
+ *mbuf_total_left, input_length);
+ return -1;
+ }
+
+ next_triplet = acc100_dma_fill_blk_type_in(desc, input,
+ in_offset, input_length,
+ seg_total_left, next_triplet);
+
+ if (unlikely(next_triplet < 0)) {
+ rte_bbdev_log(ERR,
+ "Mismatch between data to process and mbuf data length in bbdev_op: %p",
+ op);
+ return -1;
+ }
+
+ if (check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
+ h_p_size = fcw->hcin_size0 + fcw->hcin_size1;
+ if (h_comp)
+ h_p_size = (h_p_size * 3 + 3) / 4;
+ desc->data_ptrs[next_triplet].address =
+ dec->harq_combined_input.offset;
+ desc->data_ptrs[next_triplet].blen = h_p_size;
+ desc->data_ptrs[next_triplet].blkid = ACC100_DMA_BLKID_IN_HARQ;
+ desc->data_ptrs[next_triplet].dma_ext = 1;
+#ifndef ACC100_EXT_MEM
+ acc100_dma_fill_blk_type_out(
+ desc,
+ op->ldpc_dec.harq_combined_input.data,
+ op->ldpc_dec.harq_combined_input.offset,
+ h_p_size,
+ next_triplet,
+ ACC100_DMA_BLKID_IN_HARQ);
+#endif
+ next_triplet++;
+ }
+
+ desc->data_ptrs[next_triplet - 1].last = 1;
+ desc->m2dlen = next_triplet;
+ *mbuf_total_left -= input_length;
+
+ next_triplet = acc100_dma_fill_blk_type_out(desc, h_output,
+ *h_out_offset, output_length >> 3, next_triplet,
+ ACC100_DMA_BLKID_OUT_HARD);
+
+ if (check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {
+ /* Pruned size of the HARQ */
+ h_p_size = fcw->hcout_size0 + fcw->hcout_size1;
+ /* Non-Pruned size of the HARQ */
+ h_np_size = fcw->hcout_offset > 0 ?
+ fcw->hcout_offset + fcw->hcout_size1 :
+ h_p_size;
+ if (h_comp) {
+ h_np_size = (h_np_size * 3 + 3) / 4;
+ h_p_size = (h_p_size * 3 + 3) / 4;
+ }
+ dec->harq_combined_output.length = h_np_size;
+ desc->data_ptrs[next_triplet].address =
+ dec->harq_combined_output.offset;
+ desc->data_ptrs[next_triplet].blen = h_p_size;
+ desc->data_ptrs[next_triplet].blkid = ACC100_DMA_BLKID_OUT_HARQ;
+ desc->data_ptrs[next_triplet].dma_ext = 1;
+#ifndef ACC100_EXT_MEM
+ acc100_dma_fill_blk_type_out(
+ desc,
+ dec->harq_combined_output.data,
+ dec->harq_combined_output.offset,
+ h_p_size,
+ next_triplet,
+ ACC100_DMA_BLKID_OUT_HARQ);
+#endif
+ next_triplet++;
+ }
+
+ *h_out_length = output_length >> 3;
+ dec->hard_output.length += *h_out_length;
+ *h_out_offset += *h_out_length;
+ desc->data_ptrs[next_triplet - 1].last = 1;
+ desc->d2mlen = next_triplet - desc->m2dlen;
+
+ desc->op_addr = op;
+
+ return 0;
+}
+
+static inline void
+acc100_dma_desc_ld_update(struct rte_bbdev_dec_op *op,
+ struct acc100_dma_req_desc *desc,
+ struct rte_mbuf *input, struct rte_mbuf *h_output,
+ uint32_t *in_offset, uint32_t *h_out_offset,
+ uint32_t *h_out_length,
+ union acc100_harq_layout_data *harq_layout)
+{
+ int next_triplet = 1; /* FCW already done */
+ desc->data_ptrs[next_triplet].address =
+ rte_pktmbuf_iova_offset(input, *in_offset);
+ next_triplet++;
+
+ if (check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
+ struct rte_bbdev_op_data hi = op->ldpc_dec.harq_combined_input;
+ desc->data_ptrs[next_triplet].address = hi.offset;
+#ifndef ACC100_EXT_MEM
+ desc->data_ptrs[next_triplet].address =
+ rte_pktmbuf_iova_offset(hi.data, hi.offset);
+#endif
+ next_triplet++;
+ }
+
+ desc->data_ptrs[next_triplet].address =
+ rte_pktmbuf_iova_offset(h_output, *h_out_offset);
+ *h_out_length = desc->data_ptrs[next_triplet].blen;
+ next_triplet++;
+
+ if (check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {
+ desc->data_ptrs[next_triplet].address =
+ op->ldpc_dec.harq_combined_output.offset;
+ /* Adjust based on previous operation */
+ struct rte_bbdev_dec_op *prev_op = desc->op_addr;
+ op->ldpc_dec.harq_combined_output.length =
+ prev_op->ldpc_dec.harq_combined_output.length;
+ int16_t hq_idx = op->ldpc_dec.harq_combined_output.offset /
+ ACC100_HARQ_OFFSET;
+ int16_t prev_hq_idx =
+ prev_op->ldpc_dec.harq_combined_output.offset
+ / ACC100_HARQ_OFFSET;
+ harq_layout[hq_idx].val = harq_layout[prev_hq_idx].val;
+#ifndef ACC100_EXT_MEM
+ struct rte_bbdev_op_data ho =
+ op->ldpc_dec.harq_combined_output;
+ desc->data_ptrs[next_triplet].address =
+ rte_pktmbuf_iova_offset(ho.data, ho.offset);
+#endif
+ next_triplet++;
+ }
+
+ op->ldpc_dec.hard_output.length += *h_out_length;
+ desc->op_addr = op;
+}
+
+
+/* Enqueue a number of operations to HW and update software rings */
+static inline void
+acc100_dma_enqueue(struct acc100_queue *q, uint16_t n,
+ struct rte_bbdev_stats *queue_stats)
+{
+ union acc100_enqueue_reg_fmt enq_req;
+#ifdef RTE_BBDEV_OFFLOAD_COST
+ uint64_t start_time = 0;
+ queue_stats->acc_offload_cycles = 0;
+#else
+ RTE_SET_USED(queue_stats);
+#endif
+
+ enq_req.val = 0;
+ /* Setting offset, 100b for 256 DMA Desc */
+ enq_req.addr_offset = ACC100_DESC_OFFSET;
+
+ /* Split ops into batches */
+ do {
+ union acc100_dma_desc *desc;
+ uint16_t enq_batch_size;
+ uint64_t offset;
+ rte_iova_t req_elem_addr;
+
+ enq_batch_size = RTE_MIN(n, MAX_ENQ_BATCH_SIZE);
+
+ /* Set flag on last descriptor in a batch */
+ desc = q->ring_addr + ((q->sw_ring_head + enq_batch_size - 1) &
+ q->sw_ring_wrap_mask);
+ desc->req.last_desc_in_batch = 1;
+
+ /* Calculate the 1st descriptor's address */
+ offset = ((q->sw_ring_head & q->sw_ring_wrap_mask) *
+ sizeof(union acc100_dma_desc));
+ req_elem_addr = q->ring_addr_iova + offset;
+
+ /* Fill enqueue struct */
+ enq_req.num_elem = enq_batch_size;
+ /* low 6 bits are not needed */
+ enq_req.req_elem_addr = (uint32_t)(req_elem_addr >> 6);
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "Req sdone", desc, sizeof(*desc));
+#endif
+ rte_bbdev_log_debug(
+ "Enqueue %u reqs (phys %#"PRIx64") to reg %p",
+ enq_batch_size,
+ req_elem_addr,
+ (void *)q->mmio_reg_enqueue);
+
+ rte_wmb();
+
+#ifdef RTE_BBDEV_OFFLOAD_COST
+ /* Start time measurement for enqueue function offload. */
+ start_time = rte_rdtsc_precise();
+#endif
+ rte_bbdev_log(DEBUG, "Debug : MMIO Enqueue");
+ mmio_write(q->mmio_reg_enqueue, enq_req.val);
+
+#ifdef RTE_BBDEV_OFFLOAD_COST
+ queue_stats->acc_offload_cycles +=
+ rte_rdtsc_precise() - start_time;
+#endif
+
+ q->aq_enqueued++;
+ q->sw_ring_head += enq_batch_size;
+ n -= enq_batch_size;
+
+ } while (n);
+
+
+}
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+/* Validates turbo encoder parameters */
+static inline int
+validate_enc_op(struct rte_bbdev_enc_op *op)
+{
+ struct rte_bbdev_op_turbo_enc *turbo_enc = &op->turbo_enc;
+ struct rte_bbdev_op_enc_turbo_cb_params *cb = NULL;
+ struct rte_bbdev_op_enc_turbo_tb_params *tb = NULL;
+ uint16_t kw, kw_neg, kw_pos;
+
+ if (op->mempool == NULL) {
+ rte_bbdev_log(ERR, "Invalid mempool pointer");
+ return -1;
+ }
+ if (turbo_enc->input.data == NULL) {
+ rte_bbdev_log(ERR, "Invalid input pointer");
+ return -1;
+ }
+ if (turbo_enc->output.data == NULL) {
+ rte_bbdev_log(ERR, "Invalid output pointer");
+ return -1;
+ }
+ if (turbo_enc->rv_index > 3) {
+ rte_bbdev_log(ERR,
+ "rv_index (%u) is out of range 0 <= value <= 3",
+ turbo_enc->rv_index);
+ return -1;
+ }
+ if (turbo_enc->code_block_mode != 0 &&
+ turbo_enc->code_block_mode != 1) {
+ rte_bbdev_log(ERR,
+ "code_block_mode (%u) is out of range 0 <= value <= 1",
+ turbo_enc->code_block_mode);
+ return -1;
+ }
+
+ if (turbo_enc->code_block_mode == 0) {
+ tb = &turbo_enc->tb_params;
+ if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE
+ || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE)
+ && tb->c_neg > 0) {
+ rte_bbdev_log(ERR,
+ "k_neg (%u) is out of range %u <= value <= %u",
+ tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE,
+ RTE_BBDEV_TURBO_MAX_CB_SIZE);
+ return -1;
+ }
+ if (tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE
+ || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) {
+ rte_bbdev_log(ERR,
+ "k_pos (%u) is out of range %u <= value <= %u",
+ tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE,
+ RTE_BBDEV_TURBO_MAX_CB_SIZE);
+ return -1;
+ }
+ if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1))
+ rte_bbdev_log(ERR,
+ "c_neg (%u) is out of range 0 <= value <= %u",
+ tb->c_neg,
+ RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1);
+ if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) {
+ rte_bbdev_log(ERR,
+ "c (%u) is out of range 1 <= value <= %u",
+ tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS);
+ return -1;
+ }
+ if (tb->cab > tb->c) {
+ rte_bbdev_log(ERR,
+ "cab (%u) is greater than c (%u)",
+ tb->cab, tb->c);
+ return -1;
+ }
+ if ((tb->ea < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->ea % 2))
+ && tb->r < tb->cab) {
+ rte_bbdev_log(ERR,
+ "ea (%u) is less than %u or it is not even",
+ tb->ea, RTE_BBDEV_TURBO_MIN_CB_SIZE);
+ return -1;
+ }
+ if ((tb->eb < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->eb % 2))
+ && tb->c > tb->cab) {
+ rte_bbdev_log(ERR,
+ "eb (%u) is less than %u or it is not even",
+ tb->eb, RTE_BBDEV_TURBO_MIN_CB_SIZE);
+ return -1;
+ }
+
+ kw_neg = 3 * RTE_ALIGN_CEIL(tb->k_neg + 4,
+ RTE_BBDEV_TURBO_C_SUBBLOCK);
+ if (tb->ncb_neg < tb->k_neg || tb->ncb_neg > kw_neg) {
+ rte_bbdev_log(ERR,
+ "ncb_neg (%u) is out of range (%u) k_neg <= value <= (%u) kw_neg",
+ tb->ncb_neg, tb->k_neg, kw_neg);
+ return -1;
+ }
+
+ kw_pos = 3 * RTE_ALIGN_CEIL(tb->k_pos + 4,
+ RTE_BBDEV_TURBO_C_SUBBLOCK);
+ if (tb->ncb_pos < tb->k_pos || tb->ncb_pos > kw_pos) {
+ rte_bbdev_log(ERR,
+ "ncb_pos (%u) is out of range (%u) k_pos <= value <= (%u) kw_pos",
+ tb->ncb_pos, tb->k_pos, kw_pos);
+ return -1;
+ }
+ if (tb->r > (tb->c - 1)) {
+ rte_bbdev_log(ERR,
+ "r (%u) is greater than c - 1 (%u)",
+ tb->r, tb->c - 1);
+ return -1;
+ }
+ } else {
+ cb = &turbo_enc->cb_params;
+ if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE
+ || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) {
+ rte_bbdev_log(ERR,
+ "k (%u) is out of range %u <= value <= %u",
+ cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE,
+ RTE_BBDEV_TURBO_MAX_CB_SIZE);
+ return -1;
+ }
+
+ if (cb->e < RTE_BBDEV_TURBO_MIN_CB_SIZE || (cb->e % 2)) {
+ rte_bbdev_log(ERR,
+ "e (%u) is less than %u or it is not even",
+ cb->e, RTE_BBDEV_TURBO_MIN_CB_SIZE);
+ return -1;
+ }
+
+ kw = RTE_ALIGN_CEIL(cb->k + 4, RTE_BBDEV_TURBO_C_SUBBLOCK) * 3;
+ if (cb->ncb < cb->k || cb->ncb > kw) {
+ rte_bbdev_log(ERR,
+ "ncb (%u) is out of range (%u) k <= value <= (%u) kw",
+ cb->ncb, cb->k, kw);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+/* Validates LDPC encoder parameters */
+static inline int
+validate_ldpc_enc_op(struct rte_bbdev_enc_op *op)
+{
+ struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc;
+
+ if (op->mempool == NULL) {
+ rte_bbdev_log(ERR, "Invalid mempool pointer");
+ return -1;
+ }
+ if (ldpc_enc->input.data == NULL) {
+ rte_bbdev_log(ERR, "Invalid input pointer");
+ return -1;
+ }
+ if (ldpc_enc->output.data == NULL) {
+ rte_bbdev_log(ERR, "Invalid output pointer");
+ return -1;
+ }
+ if (ldpc_enc->input.length >
+ RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) {
+ rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d",
+ ldpc_enc->input.length,
+ RTE_BBDEV_LDPC_MAX_CB_SIZE);
+ return -1;
+ }
+ if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) {
+ rte_bbdev_log(ERR,
+ "BG (%u) is out of range 1 <= value <= 2",
+ ldpc_enc->basegraph);
+ return -1;
+ }
+ if (ldpc_enc->rv_index > 3) {
+ rte_bbdev_log(ERR,
+ "rv_index (%u) is out of range 0 <= value <= 3",
+ ldpc_enc->rv_index);
+ return -1;
+ }
+ if (ldpc_enc->code_block_mode > 1) {
+ rte_bbdev_log(ERR,
+ "code_block_mode (%u) is out of range 0 <= value <= 1",
+ ldpc_enc->code_block_mode);
+ return -1;
+ }
+ int K = (ldpc_enc->basegraph == 1 ? 22 : 10) * ldpc_enc->z_c;
+ if (ldpc_enc->n_filler >= K) {
+ rte_bbdev_log(ERR,
+ "K and F are not compatible %u %u",
+ K, ldpc_enc->n_filler);
+ return -1;
+ }
+ return 0;
+}
+
+/* Validates LDPC decoder parameters */
+static inline int
+validate_ldpc_dec_op(struct rte_bbdev_dec_op *op)
+{
+ struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec;
+
+ if (op->mempool == NULL) {
+ rte_bbdev_log(ERR, "Invalid mempool pointer");
+ return -1;
+ }
+ if ((ldpc_dec->basegraph > 2) || (ldpc_dec->basegraph == 0)) {
+ rte_bbdev_log(ERR,
+ "BG (%u) is out of range 1 <= value <= 2",
+ ldpc_dec->basegraph);
+ return -1;
+ }
+ if (ldpc_dec->iter_max == 0) {
+ rte_bbdev_log(ERR,
+ "iter_max (%u) is equal to 0",
+ ldpc_dec->iter_max);
+ return -1;
+ }
+ if (ldpc_dec->rv_index > 3) {
+ rte_bbdev_log(ERR,
+ "rv_index (%u) is out of range 0 <= value <= 3",
+ ldpc_dec->rv_index);
+ return -1;
+ }
+ if (ldpc_dec->code_block_mode > 1) {
+ rte_bbdev_log(ERR,
+ "code_block_mode (%u) is out of range 0 <= value <= 1",
+ ldpc_dec->code_block_mode);
+ return -1;
+ }
+ int K = (ldpc_dec->basegraph == 1 ? 22 : 10) * ldpc_dec->z_c;
+ if (ldpc_dec->n_filler >= K) {
+ rte_bbdev_log(ERR,
+ "K and F are not compatible %u %u",
+ K, ldpc_dec->n_filler);
+ return -1;
+ }
+ return 0;
+}
+#endif
+
+/* Enqueue one encode operations for ACC100 device in CB mode */
+static inline int
+enqueue_enc_one_op_cb(struct acc100_queue *q, struct rte_bbdev_enc_op *op,
+ uint16_t total_enqueued_cbs)
+{
+ union acc100_dma_desc *desc = NULL;
+ int ret;
+ uint32_t in_offset, out_offset, out_length, mbuf_total_left,
+ seg_total_left;
+ struct rte_mbuf *input, *output_head, *output;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ /* Validate op structure */
+ if (validate_enc_op(op) == -1) {
+ rte_bbdev_log(ERR, "Turbo encoder validation failed");
+ return -EINVAL;
+ }
+#endif
+
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ acc100_fcw_te_fill(op, &desc->req.fcw_te);
+
+ input = op->turbo_enc.input.data;
+ output_head = output = op->turbo_enc.output.data;
+ in_offset = op->turbo_enc.input.offset;
+ out_offset = op->turbo_enc.output.offset;
+ out_length = 0;
+ mbuf_total_left = op->turbo_enc.input.length;
+ seg_total_left = rte_pktmbuf_data_len(op->turbo_enc.input.data)
+ - in_offset;
+
+ ret = acc100_dma_desc_te_fill(op, &desc->req, &input, output,
+ &in_offset, &out_offset, &out_length, &mbuf_total_left,
+ &seg_total_left, 0);
+
+ if (unlikely(ret < 0))
+ return ret;
+
+ mbuf_append(output_head, output, out_length);
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "FCW", &desc->req.fcw_te,
+ sizeof(desc->req.fcw_te) - 8);
+ rte_memdump(stderr, "Req Desc.", desc, sizeof(*desc));
+ if (check_mbuf_total_left(mbuf_total_left) != 0)
+ return -EINVAL;
+#endif
+ /* One CB (one op) was successfully prepared to enqueue */
+ return 1;
+}
+
+/* Enqueue one encode operations for ACC100 device in CB mode */
+static inline int
+enqueue_ldpc_enc_n_op_cb(struct acc100_queue *q, struct rte_bbdev_enc_op **ops,
+ uint16_t total_enqueued_cbs, int16_t num)
+{
+ union acc100_dma_desc *desc = NULL;
+ uint32_t out_length;
+ struct rte_mbuf *output_head, *output;
+ int i, next_triplet;
+ uint16_t in_length_in_bytes;
+ struct rte_bbdev_op_ldpc_enc *enc = &ops[0]->ldpc_enc;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ /* Validate op structure */
+ if (validate_ldpc_enc_op(ops[0]) == -1) {
+ rte_bbdev_log(ERR, "LDPC encoder validation failed");
+ return -EINVAL;
+ }
+#endif
+
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ acc100_fcw_le_fill(ops[0], &desc->req.fcw_le, num);
+
+ /** This could be done at polling */
+ acc100_header_init(&desc->req);
+ desc->req.numCBs = num;
+
+ in_length_in_bytes = ops[0]->ldpc_enc.input.data->data_len;
+ out_length = (enc->cb_params.e + 7) >> 3;
+ desc->req.m2dlen = 1 + num;
+ desc->req.d2mlen = num;
+ next_triplet = 1;
+
+ for (i = 0; i < num; i++) {
+ desc->req.data_ptrs[next_triplet].address =
+ rte_pktmbuf_iova_offset(ops[i]->ldpc_enc.input.data, 0);
+ desc->req.data_ptrs[next_triplet].blen = in_length_in_bytes;
+ next_triplet++;
+ desc->req.data_ptrs[next_triplet].address =
+ rte_pktmbuf_iova_offset(
+ ops[i]->ldpc_enc.output.data, 0);
+ desc->req.data_ptrs[next_triplet].blen = out_length;
+ next_triplet++;
+ ops[i]->ldpc_enc.output.length = out_length;
+ output_head = output = ops[i]->ldpc_enc.output.data;
+ mbuf_append(output_head, output, out_length);
+ output->data_len = out_length;
+ }
+
+ desc->req.op_addr = ops[0];
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "FCW", &desc->req.fcw_le,
+ sizeof(desc->req.fcw_le) - 8);
+ rte_memdump(stderr, "Req Desc.", desc, sizeof(*desc));
+#endif
+
+ /* One CB (one op) was successfully prepared to enqueue */
+ return num;
+}
+
+/* Enqueue one encode operations for ACC100 device in CB mode */
+static inline int
+enqueue_ldpc_enc_one_op_cb(struct acc100_queue *q, struct rte_bbdev_enc_op *op,
+ uint16_t total_enqueued_cbs)
+{
+ union acc100_dma_desc *desc = NULL;
+ int ret;
+ uint32_t in_offset, out_offset, out_length, mbuf_total_left,
+ seg_total_left;
+ struct rte_mbuf *input, *output_head, *output;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ /* Validate op structure */
+ if (validate_ldpc_enc_op(op) == -1) {
+ rte_bbdev_log(ERR, "LDPC encoder validation failed");
+ return -EINVAL;
+ }
+#endif
+
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ acc100_fcw_le_fill(op, &desc->req.fcw_le, 1);
+
+ input = op->ldpc_enc.input.data;
+ output_head = output = op->ldpc_enc.output.data;
+ in_offset = op->ldpc_enc.input.offset;
+ out_offset = op->ldpc_enc.output.offset;
+ out_length = 0;
+ mbuf_total_left = op->ldpc_enc.input.length;
+ seg_total_left = rte_pktmbuf_data_len(op->ldpc_enc.input.data)
+ - in_offset;
+
+ ret = acc100_dma_desc_le_fill(op, &desc->req, &input, output,
+ &in_offset, &out_offset, &out_length, &mbuf_total_left,
+ &seg_total_left);
+
+ if (unlikely(ret < 0))
+ return ret;
+
+ mbuf_append(output_head, output, out_length);
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "FCW", &desc->req.fcw_le,
+ sizeof(desc->req.fcw_le) - 8);
+ rte_memdump(stderr, "Req Desc.", desc, sizeof(*desc));
+
+ if (check_mbuf_total_left(mbuf_total_left) != 0)
+ return -EINVAL;
+#endif
+ /* One CB (one op) was successfully prepared to enqueue */
+ return 1;
+}
+
+
+/* Enqueue one encode operations for ACC100 device in TB mode. */
+static inline int
+enqueue_enc_one_op_tb(struct acc100_queue *q, struct rte_bbdev_enc_op *op,
+ uint16_t total_enqueued_cbs, uint8_t cbs_in_tb)
+{
+ union acc100_dma_desc *desc = NULL;
+ int ret;
+ uint8_t r, c;
+ uint32_t in_offset, out_offset, out_length, mbuf_total_left,
+ seg_total_left;
+ struct rte_mbuf *input, *output_head, *output;
+ uint16_t current_enqueued_cbs = 0;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ /* Validate op structure */
+ if (validate_enc_op(op) == -1) {
+ rte_bbdev_log(ERR, "Turbo encoder validation failed");
+ return -EINVAL;
+ }
+#endif
+
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ uint64_t fcw_offset = (desc_idx << 8) + ACC100_DESC_FCW_OFFSET;
+ acc100_fcw_te_fill(op, &desc->req.fcw_te);
+
+ input = op->turbo_enc.input.data;
+ output_head = output = op->turbo_enc.output.data;
+ in_offset = op->turbo_enc.input.offset;
+ out_offset = op->turbo_enc.output.offset;
+ out_length = 0;
+ mbuf_total_left = op->turbo_enc.input.length;
+
+ c = op->turbo_enc.tb_params.c;
+ r = op->turbo_enc.tb_params.r;
+
+ while (mbuf_total_left > 0 && r < c) {
+ seg_total_left = rte_pktmbuf_data_len(input) - in_offset;
+ /* Set up DMA descriptor */
+ desc = q->ring_addr + ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc->req.data_ptrs[0].address = q->ring_addr_iova + fcw_offset;
+ desc->req.data_ptrs[0].blen = ACC100_FCW_TE_BLEN;
+
+ ret = acc100_dma_desc_te_fill(op, &desc->req, &input, output,
+ &in_offset, &out_offset, &out_length,
+ &mbuf_total_left, &seg_total_left, r);
+ if (unlikely(ret < 0))
+ return ret;
+ mbuf_append(output_head, output, out_length);
+
+ /* Set total number of CBs in TB */
+ desc->req.cbs_in_tb = cbs_in_tb;
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "FCW", &desc->req.fcw_te,
+ sizeof(desc->req.fcw_te) - 8);
+ rte_memdump(stderr, "Req Desc.", desc, sizeof(*desc));
+#endif
+
+ if (seg_total_left == 0) {
+ /* Go to the next mbuf */
+ input = input->next;
+ in_offset = 0;
+ output = output->next;
+ out_offset = 0;
+ }
+
+ total_enqueued_cbs++;
+ current_enqueued_cbs++;
+ r++;
+ }
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (check_mbuf_total_left(mbuf_total_left) != 0)
+ return -EINVAL;
+#endif
+
+ /* Set SDone on last CB descriptor for TB mode. */
+ desc->req.sdone_enable = 1;
+ desc->req.irq_enable = q->irq_enable;
+
+ return current_enqueued_cbs;
+}
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+/* Validates turbo decoder parameters */
+static inline int
+validate_dec_op(struct rte_bbdev_dec_op *op)
+{
+ struct rte_bbdev_op_turbo_dec *turbo_dec = &op->turbo_dec;
+ struct rte_bbdev_op_dec_turbo_cb_params *cb = NULL;
+ struct rte_bbdev_op_dec_turbo_tb_params *tb = NULL;
+
+ if (op->mempool == NULL) {
+ rte_bbdev_log(ERR, "Invalid mempool pointer");
+ return -1;
+ }
+ if (turbo_dec->input.data == NULL) {
+ rte_bbdev_log(ERR, "Invalid input pointer");
+ return -1;
+ }
+ if (turbo_dec->hard_output.data == NULL) {
+ rte_bbdev_log(ERR, "Invalid hard_output pointer");
+ return -1;
+ }
+ if (check_bit(turbo_dec->op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
+ turbo_dec->soft_output.data == NULL) {
+ rte_bbdev_log(ERR, "Invalid soft_output pointer");
+ return -1;
+ }
+ if (turbo_dec->rv_index > 3) {
+ rte_bbdev_log(ERR,
+ "rv_index (%u) is out of range 0 <= value <= 3",
+ turbo_dec->rv_index);
+ return -1;
+ }
+ if (turbo_dec->iter_min < 1) {
+ rte_bbdev_log(ERR,
+ "iter_min (%u) is less than 1",
+ turbo_dec->iter_min);
+ return -1;
+ }
+ if (turbo_dec->iter_max <= 2) {
+ rte_bbdev_log(ERR,
+ "iter_max (%u) is less than or equal to 2",
+ turbo_dec->iter_max);
+ return -1;
+ }
+ if (turbo_dec->iter_min > turbo_dec->iter_max) {
+ rte_bbdev_log(ERR,
+ "iter_min (%u) is greater than iter_max (%u)",
+ turbo_dec->iter_min, turbo_dec->iter_max);
+ return -1;
+ }
+ if (turbo_dec->code_block_mode != 0 &&
+ turbo_dec->code_block_mode != 1) {
+ rte_bbdev_log(ERR,
+ "code_block_mode (%u) is out of range 0 <= value <= 1",
+ turbo_dec->code_block_mode);
+ return -1;
+ }
+
+ if (turbo_dec->code_block_mode == 0) {
+ tb = &turbo_dec->tb_params;
+ if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE
+ || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE)
+ && tb->c_neg > 0) {
+ rte_bbdev_log(ERR,
+ "k_neg (%u) is out of range %u <= value <= %u",
+ tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE,
+ RTE_BBDEV_TURBO_MAX_CB_SIZE);
+ return -1;
+ }
+ if ((tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE
+ || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE)
+ && tb->c > tb->c_neg) {
+ rte_bbdev_log(ERR,
+ "k_pos (%u) is out of range %u <= value <= %u",
+ tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE,
+ RTE_BBDEV_TURBO_MAX_CB_SIZE);
+ return -1;
+ }
+ if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1))
+ rte_bbdev_log(ERR,
+ "c_neg (%u) is out of range 0 <= value <= %u",
+ tb->c_neg,
+ RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1);
+ if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) {
+ rte_bbdev_log(ERR,
+ "c (%u) is out of range 1 <= value <= %u",
+ tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS);
+ return -1;
+ }
+ if (tb->cab > tb->c) {
+ rte_bbdev_log(ERR,
+ "cab (%u) is greater than c (%u)",
+ tb->cab, tb->c);
+ return -1;
+ }
+ if (check_bit(turbo_dec->op_flags, RTE_BBDEV_TURBO_EQUALIZER) &&
+ (tb->ea < RTE_BBDEV_TURBO_MIN_CB_SIZE
+ || (tb->ea % 2))
+ && tb->cab > 0) {
+ rte_bbdev_log(ERR,
+ "ea (%u) is less than %u or it is not even",
+ tb->ea, RTE_BBDEV_TURBO_MIN_CB_SIZE);
+ return -1;
+ }
+ if (check_bit(turbo_dec->op_flags, RTE_BBDEV_TURBO_EQUALIZER) &&
+ (tb->eb < RTE_BBDEV_TURBO_MIN_CB_SIZE
+ || (tb->eb % 2))
+ && tb->c > tb->cab) {
+ rte_bbdev_log(ERR,
+ "eb (%u) is less than %u or it is not even",
+ tb->eb, RTE_BBDEV_TURBO_MIN_CB_SIZE);
+ }
+ } else {
+ cb = &turbo_dec->cb_params;
+ if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE
+ || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) {
+ rte_bbdev_log(ERR,
+ "k (%u) is out of range %u <= value <= %u",
+ cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE,
+ RTE_BBDEV_TURBO_MAX_CB_SIZE);
+ return -1;
+ }
+ if (check_bit(turbo_dec->op_flags, RTE_BBDEV_TURBO_EQUALIZER) &&
+ (cb->e < RTE_BBDEV_TURBO_MIN_CB_SIZE ||
+ (cb->e % 2))) {
+ rte_bbdev_log(ERR,
+ "e (%u) is less than %u or it is not even",
+ cb->e, RTE_BBDEV_TURBO_MIN_CB_SIZE);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+#endif
+
+/** Enqueue one decode operations for ACC100 device in CB mode */
+static inline int
+enqueue_dec_one_op_cb(struct acc100_queue *q, struct rte_bbdev_dec_op *op,
+ uint16_t total_enqueued_cbs)
+{
+ union acc100_dma_desc *desc = NULL;
+ int ret;
+ uint32_t in_offset, h_out_offset, s_out_offset, s_out_length,
+ h_out_length, mbuf_total_left, seg_total_left;
+ struct rte_mbuf *input, *h_output_head, *h_output,
+ *s_output_head, *s_output;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ /* Validate op structure */
+ if (validate_dec_op(op) == -1) {
+ rte_bbdev_log(ERR, "Turbo decoder validation failed");
+ return -EINVAL;
+ }
+#endif
+
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ acc100_fcw_td_fill(op, &desc->req.fcw_td);
+
+ input = op->turbo_dec.input.data;
+ h_output_head = h_output = op->turbo_dec.hard_output.data;
+ s_output_head = s_output = op->turbo_dec.soft_output.data;
+ in_offset = op->turbo_dec.input.offset;
+ h_out_offset = op->turbo_dec.hard_output.offset;
+ s_out_offset = op->turbo_dec.soft_output.offset;
+ h_out_length = s_out_length = 0;
+ mbuf_total_left = op->turbo_dec.input.length;
+ seg_total_left = rte_pktmbuf_data_len(input) - in_offset;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (unlikely(input == NULL)) {
+ rte_bbdev_log(ERR, "Invalid mbuf pointer");
+ return -EFAULT;
+ }
+#endif
+
+ /* Set up DMA descriptor */
+ desc = q->ring_addr + ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+
+ ret = acc100_dma_desc_td_fill(op, &desc->req, &input, h_output,
+ s_output, &in_offset, &h_out_offset, &s_out_offset,
+ &h_out_length, &s_out_length, &mbuf_total_left,
+ &seg_total_left, 0);
+
+ if (unlikely(ret < 0))
+ return ret;
+
+ /* Hard output */
+ mbuf_append(h_output_head, h_output, h_out_length);
+
+ /* Soft output */
+ if (check_bit(op->turbo_dec.op_flags, RTE_BBDEV_TURBO_SOFT_OUTPUT))
+ mbuf_append(s_output_head, s_output, s_out_length);
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "FCW", &desc->req.fcw_td,
+ sizeof(desc->req.fcw_td) - 8);
+ rte_memdump(stderr, "Req Desc.", desc, sizeof(*desc));
+ if (check_mbuf_total_left(mbuf_total_left) != 0)
+ return -EINVAL;
+#endif
+
+ /* One CB (one op) was successfully prepared to enqueue */
+ return 1;
+}
+
+static inline int
+harq_loopback(struct acc100_queue *q, struct rte_bbdev_dec_op *op,
+ uint16_t total_enqueued_cbs) {
+ struct acc100_fcw_ld *fcw;
+ union acc100_dma_desc *desc;
+ int next_triplet = 1;
+ struct rte_mbuf *hq_output_head, *hq_output;
+ uint16_t harq_dma_length_in, harq_dma_length_out;
+ uint16_t harq_in_length = op->ldpc_dec.harq_combined_input.length;
+ if (harq_in_length == 0) {
+ rte_bbdev_log(ERR, "Loopback of invalid null size\n");
+ return -EINVAL;
+ }
+
+ int h_comp = check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION
+ ) ? 1 : 0;
+ if (h_comp == 1) {
+ harq_in_length = harq_in_length * 8 / 6;
+ harq_in_length = RTE_ALIGN(harq_in_length, 64);
+ harq_dma_length_in = harq_in_length * 6 / 8;
+ } else {
+ harq_in_length = RTE_ALIGN(harq_in_length, 64);
+ harq_dma_length_in = harq_in_length;
+ }
+ harq_dma_length_out = harq_dma_length_in;
+
+ bool ddr_mem_in = check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE);
+ union acc100_harq_layout_data *harq_layout = q->d->harq_layout;
+ uint16_t harq_index = (ddr_mem_in ?
+ op->ldpc_dec.harq_combined_input.offset :
+ op->ldpc_dec.harq_combined_output.offset)
+ / ACC100_HARQ_OFFSET;
+
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ fcw = &desc->req.fcw_ld;
+ /* Set the FCW from loopback into DDR */
+ memset(fcw, 0, sizeof(struct acc100_fcw_ld));
+ fcw->FCWversion = ACC100_FCW_VER;
+ fcw->qm = 2;
+ fcw->Zc = 384;
+ if (harq_in_length < 16 * ACC100_N_ZC_1)
+ fcw->Zc = 16;
+ fcw->ncb = fcw->Zc * ACC100_N_ZC_1;
+ fcw->rm_e = 2;
+ fcw->hcin_en = 1;
+ fcw->hcout_en = 1;
+
+ rte_bbdev_log(DEBUG, "Loopback IN %d Index %d offset %d length %d %d\n",
+ ddr_mem_in, harq_index,
+ harq_layout[harq_index].offset, harq_in_length,
+ harq_dma_length_in);
+
+ if (ddr_mem_in && (harq_layout[harq_index].offset > 0)) {
+ fcw->hcin_size0 = harq_layout[harq_index].size0;
+ fcw->hcin_offset = harq_layout[harq_index].offset;
+ fcw->hcin_size1 = harq_in_length - fcw->hcin_offset;
+ harq_dma_length_in = (fcw->hcin_size0 + fcw->hcin_size1);
+ if (h_comp == 1)
+ harq_dma_length_in = harq_dma_length_in * 6 / 8;
+ } else {
+ fcw->hcin_size0 = harq_in_length;
+ }
+ harq_layout[harq_index].val = 0;
+ rte_bbdev_log(DEBUG, "Loopback FCW Config %d %d %d\n",
+ fcw->hcin_size0, fcw->hcin_offset, fcw->hcin_size1);
+ fcw->hcout_size0 = harq_in_length;
+ fcw->hcin_decomp_mode = h_comp;
+ fcw->hcout_comp_mode = h_comp;
+ fcw->gain_i = 1;
+ fcw->gain_h = 1;
+
+ /* Set the prefix of descriptor. This could be done at polling */
+ acc100_header_init(&desc->req);
+
+ /* Null LLR input for Decoder */
+ desc->req.data_ptrs[next_triplet].address =
+ q->lb_in_addr_iova;
+ desc->req.data_ptrs[next_triplet].blen = 2;
+ desc->req.data_ptrs[next_triplet].blkid = ACC100_DMA_BLKID_IN;
+ desc->req.data_ptrs[next_triplet].last = 0;
+ desc->req.data_ptrs[next_triplet].dma_ext = 0;
+ next_triplet++;
+
+ /* HARQ Combine input from either Memory interface */
+ if (!ddr_mem_in) {
+ next_triplet = acc100_dma_fill_blk_type_out(&desc->req,
+ op->ldpc_dec.harq_combined_input.data,
+ op->ldpc_dec.harq_combined_input.offset,
+ harq_dma_length_in,
+ next_triplet,
+ ACC100_DMA_BLKID_IN_HARQ);
+ } else {
+ desc->req.data_ptrs[next_triplet].address =
+ op->ldpc_dec.harq_combined_input.offset;
+ desc->req.data_ptrs[next_triplet].blen =
+ harq_dma_length_in;
+ desc->req.data_ptrs[next_triplet].blkid =
+ ACC100_DMA_BLKID_IN_HARQ;
+ desc->req.data_ptrs[next_triplet].dma_ext = 1;
+ next_triplet++;
+ }
+ desc->req.data_ptrs[next_triplet - 1].last = 1;
+ desc->req.m2dlen = next_triplet;
+
+ /* Dropped decoder hard output */
+ desc->req.data_ptrs[next_triplet].address =
+ q->lb_out_addr_iova;
+ desc->req.data_ptrs[next_triplet].blen = ACC100_BYTES_IN_WORD;
+ desc->req.data_ptrs[next_triplet].blkid = ACC100_DMA_BLKID_OUT_HARD;
+ desc->req.data_ptrs[next_triplet].last = 0;
+ desc->req.data_ptrs[next_triplet].dma_ext = 0;
+ next_triplet++;
+
+ /* HARQ Combine output to either Memory interface */
+ if (check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE
+ )) {
+ desc->req.data_ptrs[next_triplet].address =
+ op->ldpc_dec.harq_combined_output.offset;
+ desc->req.data_ptrs[next_triplet].blen =
+ harq_dma_length_out;
+ desc->req.data_ptrs[next_triplet].blkid =
+ ACC100_DMA_BLKID_OUT_HARQ;
+ desc->req.data_ptrs[next_triplet].dma_ext = 1;
+ next_triplet++;
+ } else {
+ hq_output_head = op->ldpc_dec.harq_combined_output.data;
+ hq_output = op->ldpc_dec.harq_combined_output.data;
+ next_triplet = acc100_dma_fill_blk_type_out(
+ &desc->req,
+ op->ldpc_dec.harq_combined_output.data,
+ op->ldpc_dec.harq_combined_output.offset,
+ harq_dma_length_out,
+ next_triplet,
+ ACC100_DMA_BLKID_OUT_HARQ);
+ /* HARQ output */
+ mbuf_append(hq_output_head, hq_output, harq_dma_length_out);
+ op->ldpc_dec.harq_combined_output.length =
+ harq_dma_length_out;
+ }
+ desc->req.data_ptrs[next_triplet - 1].last = 1;
+ desc->req.d2mlen = next_triplet - desc->req.m2dlen;
+ desc->req.op_addr = op;
+
+ /* One CB (one op) was successfully prepared to enqueue */
+ return 1;
+}
+
+/** Enqueue one decode operations for ACC100 device in CB mode */
+static inline int
+enqueue_ldpc_dec_one_op_cb(struct acc100_queue *q, struct rte_bbdev_dec_op *op,
+ uint16_t total_enqueued_cbs, bool same_op)
+{
+ int ret;
+ if (unlikely(check_bit(op->ldpc_dec.op_flags,
+ RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK))) {
+ ret = harq_loopback(q, op, total_enqueued_cbs);
+ return ret;
+ }
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ /* Validate op structure */
+ if (validate_ldpc_dec_op(op) == -1) {
+ rte_bbdev_log(ERR, "LDPC decoder validation failed");
+ return -EINVAL;
+ }
+#endif
+ union acc100_dma_desc *desc;
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ struct rte_mbuf *input, *h_output_head, *h_output;
+ uint32_t in_offset, h_out_offset, mbuf_total_left, h_out_length = 0;
+ input = op->ldpc_dec.input.data;
+ h_output_head = h_output = op->ldpc_dec.hard_output.data;
+ in_offset = op->ldpc_dec.input.offset;
+ h_out_offset = op->ldpc_dec.hard_output.offset;
+ mbuf_total_left = op->ldpc_dec.input.length;
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (unlikely(input == NULL)) {
+ rte_bbdev_log(ERR, "Invalid mbuf pointer");
+ return -EFAULT;
+ }
+#endif
+ union acc100_harq_layout_data *harq_layout = q->d->harq_layout;
+
+ if (same_op) {
+ union acc100_dma_desc *prev_desc;
+ desc_idx = ((q->sw_ring_head + total_enqueued_cbs - 1)
+ & q->sw_ring_wrap_mask);
+ prev_desc = q->ring_addr + desc_idx;
+ uint8_t *prev_ptr = (uint8_t *) prev_desc;
+ uint8_t *new_ptr = (uint8_t *) desc;
+ /* Copy first 4 words and BDESCs */
+ rte_memcpy(new_ptr, prev_ptr, ACC100_5GUL_SIZE_0);
+ rte_memcpy(new_ptr + ACC100_5GUL_OFFSET_0,
+ prev_ptr + ACC100_5GUL_OFFSET_0,
+ ACC100_5GUL_SIZE_1);
+ desc->req.op_addr = prev_desc->req.op_addr;
+ /* Copy FCW */
+ rte_memcpy(new_ptr + ACC100_DESC_FCW_OFFSET,
+ prev_ptr + ACC100_DESC_FCW_OFFSET,
+ ACC100_FCW_LD_BLEN);
+ acc100_dma_desc_ld_update(op, &desc->req, input, h_output,
+ &in_offset, &h_out_offset,
+ &h_out_length, harq_layout);
+ } else {
+ struct acc100_fcw_ld *fcw;
+ uint32_t seg_total_left;
+ fcw = &desc->req.fcw_ld;
+ acc100_fcw_ld_fill(op, fcw, harq_layout);
+
+ /* Special handling when overusing mbuf */
+ if (fcw->rm_e < ACC100_MAX_E_MBUF)
+ seg_total_left = rte_pktmbuf_data_len(input)
+ - in_offset;
+ else
+ seg_total_left = fcw->rm_e;
+
+ ret = acc100_dma_desc_ld_fill(op, &desc->req, &input, h_output,
+ &in_offset, &h_out_offset,
+ &h_out_length, &mbuf_total_left,
+ &seg_total_left, fcw);
+ if (unlikely(ret < 0))
+ return ret;
+ }
+
+ /* Hard output */
+ mbuf_append(h_output_head, h_output, h_out_length);
+#ifndef ACC100_EXT_MEM
+ if (op->ldpc_dec.harq_combined_output.length > 0) {
+ /* Push the HARQ output into host memory */
+ struct rte_mbuf *hq_output_head, *hq_output;
+ hq_output_head = op->ldpc_dec.harq_combined_output.data;
+ hq_output = op->ldpc_dec.harq_combined_output.data;
+ mbuf_append(hq_output_head, hq_output,
+ op->ldpc_dec.harq_combined_output.length);
+ }
+#endif
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "FCW", &desc->req.fcw_ld,
+ sizeof(desc->req.fcw_ld) - 8);
+ rte_memdump(stderr, "Req Desc.", desc, sizeof(*desc));
+#endif
+
+ /* One CB (one op) was successfully prepared to enqueue */
+ return 1;
+}
+
+
+/* Enqueue one decode operations for ACC100 device in TB mode */
+static inline int
+enqueue_ldpc_dec_one_op_tb(struct acc100_queue *q, struct rte_bbdev_dec_op *op,
+ uint16_t total_enqueued_cbs, uint8_t cbs_in_tb)
+{
+ union acc100_dma_desc *desc = NULL;
+ int ret;
+ uint8_t r, c;
+ uint32_t in_offset, h_out_offset,
+ h_out_length, mbuf_total_left, seg_total_left;
+ struct rte_mbuf *input, *h_output_head, *h_output;
+ uint16_t current_enqueued_cbs = 0;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ /* Validate op structure */
+ if (validate_ldpc_dec_op(op) == -1) {
+ rte_bbdev_log(ERR, "LDPC decoder validation failed");
+ return -EINVAL;
+ }
+#endif
+
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ uint64_t fcw_offset = (desc_idx << 8) + ACC100_DESC_FCW_OFFSET;
+ union acc100_harq_layout_data *harq_layout = q->d->harq_layout;
+ acc100_fcw_ld_fill(op, &desc->req.fcw_ld, harq_layout);
+
+ input = op->ldpc_dec.input.data;
+ h_output_head = h_output = op->ldpc_dec.hard_output.data;
+ in_offset = op->ldpc_dec.input.offset;
+ h_out_offset = op->ldpc_dec.hard_output.offset;
+ h_out_length = 0;
+ mbuf_total_left = op->ldpc_dec.input.length;
+ c = op->ldpc_dec.tb_params.c;
+ r = op->ldpc_dec.tb_params.r;
+
+ while (mbuf_total_left > 0 && r < c) {
+
+ seg_total_left = rte_pktmbuf_data_len(input) - in_offset;
+
+ /* Set up DMA descriptor */
+ desc = q->ring_addr + ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc->req.data_ptrs[0].address = q->ring_addr_iova + fcw_offset;
+ desc->req.data_ptrs[0].blen = ACC100_FCW_LD_BLEN;
+ ret = acc100_dma_desc_ld_fill(op, &desc->req, &input,
+ h_output, &in_offset, &h_out_offset,
+ &h_out_length,
+ &mbuf_total_left, &seg_total_left,
+ &desc->req.fcw_ld);
+
+ if (unlikely(ret < 0))
+ return ret;
+
+ /* Hard output */
+ mbuf_append(h_output_head, h_output, h_out_length);
+
+ /* Set total number of CBs in TB */
+ desc->req.cbs_in_tb = cbs_in_tb;
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "FCW", &desc->req.fcw_td,
+ sizeof(desc->req.fcw_td) - 8);
+ rte_memdump(stderr, "Req Desc.", desc, sizeof(*desc));
+#endif
+
+ if (seg_total_left == 0) {
+ /* Go to the next mbuf */
+ input = input->next;
+ in_offset = 0;
+ h_output = h_output->next;
+ h_out_offset = 0;
+ }
+ total_enqueued_cbs++;
+ current_enqueued_cbs++;
+ r++;
+ }
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (check_mbuf_total_left(mbuf_total_left) != 0)
+ return -EINVAL;
+#endif
+ /* Set SDone on last CB descriptor for TB mode */
+ desc->req.sdone_enable = 1;
+ desc->req.irq_enable = q->irq_enable;
+
+ return current_enqueued_cbs;
+}
+
+/* Enqueue one decode operations for ACC100 device in TB mode */
+static inline int
+enqueue_dec_one_op_tb(struct acc100_queue *q, struct rte_bbdev_dec_op *op,
+ uint16_t total_enqueued_cbs, uint8_t cbs_in_tb)
+{
+ union acc100_dma_desc *desc = NULL;
+ int ret;
+ uint8_t r, c;
+ uint32_t in_offset, h_out_offset, s_out_offset, s_out_length,
+ h_out_length, mbuf_total_left, seg_total_left;
+ struct rte_mbuf *input, *h_output_head, *h_output,
+ *s_output_head, *s_output;
+ uint16_t current_enqueued_cbs = 0;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ /* Validate op structure */
+ if (validate_dec_op(op) == -1) {
+ rte_bbdev_log(ERR, "Turbo decoder validation failed");
+ return -EINVAL;
+ }
+#endif
+
+ uint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc = q->ring_addr + desc_idx;
+ uint64_t fcw_offset = (desc_idx << 8) + ACC100_DESC_FCW_OFFSET;
+ acc100_fcw_td_fill(op, &desc->req.fcw_td);
+
+ input = op->turbo_dec.input.data;
+ h_output_head = h_output = op->turbo_dec.hard_output.data;
+ s_output_head = s_output = op->turbo_dec.soft_output.data;
+ in_offset = op->turbo_dec.input.offset;
+ h_out_offset = op->turbo_dec.hard_output.offset;
+ s_out_offset = op->turbo_dec.soft_output.offset;
+ h_out_length = s_out_length = 0;
+ mbuf_total_left = op->turbo_dec.input.length;
+ c = op->turbo_dec.tb_params.c;
+ r = op->turbo_dec.tb_params.r;
+
+ while (mbuf_total_left > 0 && r < c) {
+
+ seg_total_left = rte_pktmbuf_data_len(input) - in_offset;
+
+ /* Set up DMA descriptor */
+ desc = q->ring_addr + ((q->sw_ring_head + total_enqueued_cbs)
+ & q->sw_ring_wrap_mask);
+ desc->req.data_ptrs[0].address = q->ring_addr_iova + fcw_offset;
+ desc->req.data_ptrs[0].blen = ACC100_FCW_TD_BLEN;
+ ret = acc100_dma_desc_td_fill(op, &desc->req, &input,
+ h_output, s_output, &in_offset, &h_out_offset,
+ &s_out_offset, &h_out_length, &s_out_length,
+ &mbuf_total_left, &seg_total_left, r);
+
+ if (unlikely(ret < 0))
+ return ret;
+
+ /* Hard output */
+ mbuf_append(h_output_head, h_output, h_out_length);
+
+ /* Soft output */
+ if (check_bit(op->turbo_dec.op_flags,
+ RTE_BBDEV_TURBO_SOFT_OUTPUT))
+ mbuf_append(s_output_head, s_output, s_out_length);
+
+ /* Set total number of CBs in TB */
+ desc->req.cbs_in_tb = cbs_in_tb;
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ rte_memdump(stderr, "FCW", &desc->req.fcw_td,
+ sizeof(desc->req.fcw_td) - 8);
+ rte_memdump(stderr, "Req Desc.", desc, sizeof(*desc));
+#endif
+
+ if (seg_total_left == 0) {
+ /* Go to the next mbuf */
+ input = input->next;
+ in_offset = 0;
+ h_output = h_output->next;
+ h_out_offset = 0;
+
+ if (check_bit(op->turbo_dec.op_flags,
+ RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
+ s_output = s_output->next;
+ s_out_offset = 0;
+ }
+ }
+
+ total_enqueued_cbs++;
+ current_enqueued_cbs++;
+ r++;
+ }
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (check_mbuf_total_left(mbuf_total_left) != 0)
+ return -EINVAL;
+#endif
+ /* Set SDone on last CB descriptor for TB mode */
+ desc->req.sdone_enable = 1;
+ desc->req.irq_enable = q->irq_enable;
+
+ return current_enqueued_cbs;
+}
+
+/* Calculates number of CBs in processed encoder TB based on 'r' and input
+ * length.
+ */
+static inline uint8_t
+get_num_cbs_in_tb_enc(struct rte_bbdev_op_turbo_enc *turbo_enc)
+{
+ uint8_t c, c_neg, r, crc24_bits = 0;
+ uint16_t k, k_neg, k_pos;
+ uint8_t cbs_in_tb = 0;
+ int32_t length;
+
+ length = turbo_enc->input.length;
+ r = turbo_enc->tb_params.r;
+ c = turbo_enc->tb_params.c;
+ c_neg = turbo_enc->tb_params.c_neg;
+ k_neg = turbo_enc->tb_params.k_neg;
+ k_pos = turbo_enc->tb_params.k_pos;
+ crc24_bits = 0;
+ if (check_bit(turbo_enc->op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH))
+ crc24_bits = 24;
+ while (length > 0 && r < c) {
+ k = (r < c_neg) ? k_neg : k_pos;
+ length -= (k - crc24_bits) >> 3;
+ r++;
+ cbs_in_tb++;
+ }
+
+ return cbs_in_tb;
+}
+
+/* Calculates number of CBs in processed decoder TB based on 'r' and input
+ * length.
+ */
+static inline uint16_t
+get_num_cbs_in_tb_dec(struct rte_bbdev_op_turbo_dec *turbo_dec)
+{
+ uint8_t c, c_neg, r = 0;
+ uint16_t kw, k, k_neg, k_pos, cbs_in_tb = 0;
+ int32_t length;
+
+ length = turbo_dec->input.length;
+ r = turbo_dec->tb_params.r;
+ c = turbo_dec->tb_params.c;
+ c_neg = turbo_dec->tb_params.c_neg;
+ k_neg = turbo_dec->tb_params.k_neg;
+ k_pos = turbo_dec->tb_params.k_pos;
+ while (length > 0 && r < c) {
+ k = (r < c_neg) ? k_neg : k_pos;
+ kw = RTE_ALIGN_CEIL(k + 4, 32) * 3;
+ length -= kw;
+ r++;
+ cbs_in_tb++;
+ }
+
+ return cbs_in_tb;
+}
+
+/* Calculates number of CBs in processed decoder TB based on 'r' and input
+ * length.
+ */
+static inline uint16_t
+get_num_cbs_in_tb_ldpc_dec(struct rte_bbdev_op_ldpc_dec *ldpc_dec)
+{
+ uint16_t r, cbs_in_tb = 0;
+ int32_t length = ldpc_dec->input.length;
+ r = ldpc_dec->tb_params.r;
+ while (length > 0 && r < ldpc_dec->tb_params.c) {
+ length -= (r < ldpc_dec->tb_params.cab) ?
+ ldpc_dec->tb_params.ea :
+ ldpc_dec->tb_params.eb;
+ r++;
+ cbs_in_tb++;
+ }
+ return cbs_in_tb;
+}
+
+/* Enqueue encode operations for ACC100 device in CB mode. */
+static uint16_t
+acc100_enqueue_enc_cb(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_enc_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ int32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;
+ uint16_t i;
+ union acc100_dma_desc *desc;
+ int ret;
+
+ for (i = 0; i < num; ++i) {
+ /* Check if there are available space for further processing */
+ if (unlikely(avail - 1 < 0))
+ break;
+ avail -= 1;
+
+ ret = enqueue_enc_one_op_cb(q, ops[i], i);
+ if (ret < 0)
+ break;
+ }
+
+ if (unlikely(i == 0))
+ return 0; /* Nothing to enqueue */
+
+ /* Set SDone in last CB in enqueued ops for CB mode*/
+ desc = q->ring_addr + ((q->sw_ring_head + i - 1)
+ & q->sw_ring_wrap_mask);
+ desc->req.sdone_enable = 1;
+ desc->req.irq_enable = q->irq_enable;
+
+ acc100_dma_enqueue(q, i, &q_data->queue_stats);
+
+ /* Update stats */
+ q_data->queue_stats.enqueued_count += i;
+ q_data->queue_stats.enqueue_err_count += num - i;
+ return i;
+}
+
+/* Check we can mux encode operations with common FCW */
+static inline bool
+check_mux(struct rte_bbdev_enc_op **ops, uint16_t num) {
+ uint16_t i;
+ if (num <= 1)
+ return false;
+ for (i = 1; i < num; ++i) {
+ /* Only mux compatible code blocks */
+ if (memcmp((uint8_t *)(&ops[i]->ldpc_enc) + ACC100_ENC_OFFSET,
+ (uint8_t *)(&ops[0]->ldpc_enc) +
+ ACC100_ENC_OFFSET,
+ ACC100_CMP_ENC_SIZE) != 0)
+ return false;
+ }
+ return true;
+}
+
+/** Enqueue encode operations for ACC100 device in CB mode. */
+static inline uint16_t
+acc100_enqueue_ldpc_enc_cb(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_enc_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ int32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;
+ uint16_t i = 0;
+ union acc100_dma_desc *desc;
+ int ret, desc_idx = 0;
+ int16_t enq, left = num;
+
+ while (left > 0) {
+ if (unlikely(avail < 1))
+ break;
+ avail--;
+ enq = RTE_MIN(left, ACC100_MUX_5GDL_DESC);
+ if (check_mux(&ops[i], enq)) {
+ ret = enqueue_ldpc_enc_n_op_cb(q, &ops[i],
+ desc_idx, enq);
+ if (ret < 0)
+ break;
+ i += enq;
+ } else {
+ ret = enqueue_ldpc_enc_one_op_cb(q, ops[i], desc_idx);
+ if (ret < 0)
+ break;
+ i++;
+ }
+ desc_idx++;
+ left = num - i;
+ }
+
+ if (unlikely(i == 0))
+ return 0; /* Nothing to enqueue */
+
+ /* Set SDone in last CB in enqueued ops for CB mode*/
+ desc = q->ring_addr + ((q->sw_ring_head + desc_idx - 1)
+ & q->sw_ring_wrap_mask);
+ desc->req.sdone_enable = 1;
+ desc->req.irq_enable = q->irq_enable;
+
+ acc100_dma_enqueue(q, desc_idx, &q_data->queue_stats);
+
+ /* Update stats */
+ q_data->queue_stats.enqueued_count += i;
+ q_data->queue_stats.enqueue_err_count += num - i;
+
+ return i;
+}
+
+/* Enqueue encode operations for ACC100 device in TB mode. */
+static uint16_t
+acc100_enqueue_enc_tb(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_enc_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ int32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;
+ uint16_t i, enqueued_cbs = 0;
+ uint8_t cbs_in_tb;
+ int ret;
+
+ for (i = 0; i < num; ++i) {
+ cbs_in_tb = get_num_cbs_in_tb_enc(&ops[i]->turbo_enc);
+ /* Check if there are available space for further processing */
+ if (unlikely(avail - cbs_in_tb < 0))
+ break;
+ avail -= cbs_in_tb;
+
+ ret = enqueue_enc_one_op_tb(q, ops[i], enqueued_cbs, cbs_in_tb);
+ if (ret < 0)
+ break;
+ enqueued_cbs += ret;
+ }
+ if (unlikely(enqueued_cbs == 0))
+ return 0; /* Nothing to enqueue */
+
+ acc100_dma_enqueue(q, enqueued_cbs, &q_data->queue_stats);
+
+ /* Update stats */
+ q_data->queue_stats.enqueued_count += i;
+ q_data->queue_stats.enqueue_err_count += num - i;
+
+ return i;
+}
+
+/* Enqueue encode operations for ACC100 device. */
+static uint16_t
+acc100_enqueue_enc(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_enc_op **ops, uint16_t num)
+{
+ if (unlikely(num == 0))
+ return 0;
+ if (ops[0]->turbo_enc.code_block_mode == 0)
+ return acc100_enqueue_enc_tb(q_data, ops, num);
+ else
+ return acc100_enqueue_enc_cb(q_data, ops, num);
+}
+
+/* Enqueue encode operations for ACC100 device. */
+static uint16_t
+acc100_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_enc_op **ops, uint16_t num)
+{
+ if (unlikely(num == 0))
+ return 0;
+ if (ops[0]->ldpc_enc.code_block_mode == 0)
+ return acc100_enqueue_enc_tb(q_data, ops, num);
+ else
+ return acc100_enqueue_ldpc_enc_cb(q_data, ops, num);
+}
+
+
+/* Enqueue decode operations for ACC100 device in CB mode */
+static uint16_t
+acc100_enqueue_dec_cb(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_dec_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ int32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;
+ uint16_t i;
+ union acc100_dma_desc *desc;
+ int ret;
+
+ for (i = 0; i < num; ++i) {
+ /* Check if there are available space for further processing */
+ if (unlikely(avail - 1 < 0))
+ break;
+ avail -= 1;
+
+ ret = enqueue_dec_one_op_cb(q, ops[i], i);
+ if (ret < 0)
+ break;
+ }
+
+ if (unlikely(i == 0))
+ return 0; /* Nothing to enqueue */
+
+ /* Set SDone in last CB in enqueued ops for CB mode*/
+ desc = q->ring_addr + ((q->sw_ring_head + i - 1)
+ & q->sw_ring_wrap_mask);
+ desc->req.sdone_enable = 1;
+ desc->req.irq_enable = q->irq_enable;
+
+ acc100_dma_enqueue(q, i, &q_data->queue_stats);
+
+ /* Update stats */
+ q_data->queue_stats.enqueued_count += i;
+ q_data->queue_stats.enqueue_err_count += num - i;
+
+ return i;
+}
+
+/* Check we can mux encode operations with common FCW */
+static inline bool
+cmp_ldpc_dec_op(struct rte_bbdev_dec_op **ops) {
+ /* Only mux compatible code blocks */
+ if (memcmp((uint8_t *)(&ops[0]->ldpc_dec) + ACC100_DEC_OFFSET,
+ (uint8_t *)(&ops[1]->ldpc_dec) +
+ ACC100_DEC_OFFSET, ACC100_CMP_DEC_SIZE) != 0) {
+ return false;
+ } else
+ return true;
+}
+
+
+/* Enqueue decode operations for ACC100 device in TB mode */
+static uint16_t
+acc100_enqueue_ldpc_dec_tb(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_dec_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ int32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;
+ uint16_t i, enqueued_cbs = 0;
+ uint8_t cbs_in_tb;
+ int ret;
+
+ for (i = 0; i < num; ++i) {
+ cbs_in_tb = get_num_cbs_in_tb_ldpc_dec(&ops[i]->ldpc_dec);
+ /* Check if there are available space for further processing */
+ if (unlikely(avail - cbs_in_tb < 0))
+ break;
+ avail -= cbs_in_tb;
+
+ ret = enqueue_ldpc_dec_one_op_tb(q, ops[i],
+ enqueued_cbs, cbs_in_tb);
+ if (ret < 0)
+ break;
+ enqueued_cbs += ret;
+ }
+
+ acc100_dma_enqueue(q, enqueued_cbs, &q_data->queue_stats);
+
+ /* Update stats */
+ q_data->queue_stats.enqueued_count += i;
+ q_data->queue_stats.enqueue_err_count += num - i;
+ return i;
+}
+
+/* Enqueue decode operations for ACC100 device in CB mode */
+static uint16_t
+acc100_enqueue_ldpc_dec_cb(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_dec_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ int32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;
+ uint16_t i;
+ union acc100_dma_desc *desc;
+ int ret;
+ bool same_op = false;
+ for (i = 0; i < num; ++i) {
+ /* Check if there are available space for further processing */
+ if (unlikely(avail < 1))
+ break;
+ avail -= 1;
+
+ if (i > 0)
+ same_op = cmp_ldpc_dec_op(&ops[i-1]);
+ rte_bbdev_log(INFO, "Op %d %d %d %d %d %d %d %d %d %d %d %d\n",
+ i, ops[i]->ldpc_dec.op_flags, ops[i]->ldpc_dec.rv_index,
+ ops[i]->ldpc_dec.iter_max, ops[i]->ldpc_dec.iter_count,
+ ops[i]->ldpc_dec.basegraph, ops[i]->ldpc_dec.z_c,
+ ops[i]->ldpc_dec.n_cb, ops[i]->ldpc_dec.q_m,
+ ops[i]->ldpc_dec.n_filler, ops[i]->ldpc_dec.cb_params.e,
+ same_op);
+ ret = enqueue_ldpc_dec_one_op_cb(q, ops[i], i, same_op);
+ if (ret < 0)
+ break;
+ }
+
+ if (unlikely(i == 0))
+ return 0; /* Nothing to enqueue */
+
+ /* Set SDone in last CB in enqueued ops for CB mode*/
+ desc = q->ring_addr + ((q->sw_ring_head + i - 1)
+ & q->sw_ring_wrap_mask);
+
+ desc->req.sdone_enable = 1;
+ desc->req.irq_enable = q->irq_enable;
+
+ acc100_dma_enqueue(q, i, &q_data->queue_stats);
+
+ /* Update stats */
+ q_data->queue_stats.enqueued_count += i;
+ q_data->queue_stats.enqueue_err_count += num - i;
+ return i;
+}
+
+
+/* Enqueue decode operations for ACC100 device in TB mode */
+static uint16_t
+acc100_enqueue_dec_tb(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_dec_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ int32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;
+ uint16_t i, enqueued_cbs = 0;
+ uint8_t cbs_in_tb;
+ int ret;
+
+ for (i = 0; i < num; ++i) {
+ cbs_in_tb = get_num_cbs_in_tb_dec(&ops[i]->turbo_dec);
+ /* Check if there are available space for further processing */
+ if (unlikely(avail - cbs_in_tb < 0))
+ break;
+ avail -= cbs_in_tb;
+
+ ret = enqueue_dec_one_op_tb(q, ops[i], enqueued_cbs, cbs_in_tb);
+ if (ret < 0)
+ break;
+ enqueued_cbs += ret;
+ }
+
+ acc100_dma_enqueue(q, enqueued_cbs, &q_data->queue_stats);
+
+ /* Update stats */
+ q_data->queue_stats.enqueued_count += i;
+ q_data->queue_stats.enqueue_err_count += num - i;
+
+ return i;
+}
+
+/* Enqueue decode operations for ACC100 device. */
+static uint16_t
+acc100_enqueue_dec(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_dec_op **ops, uint16_t num)
+{
+ if (unlikely(num == 0))
+ return 0;
+ if (ops[0]->turbo_dec.code_block_mode == 0)
+ return acc100_enqueue_dec_tb(q_data, ops, num);
+ else
+ return acc100_enqueue_dec_cb(q_data, ops, num);
+}
+
+/* Enqueue decode operations for ACC100 device. */
+static uint16_t
+acc100_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_dec_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ int32_t aq_avail = q->aq_depth +
+ (q->aq_dequeued - q->aq_enqueued) / 128;
+
+ if (unlikely((aq_avail == 0) || (num == 0)))
+ return 0;
+
+ if (ops[0]->ldpc_dec.code_block_mode == 0)
+ return acc100_enqueue_ldpc_dec_tb(q_data, ops, num);
+ else
+ return acc100_enqueue_ldpc_dec_cb(q_data, ops, num);
+}
+
+
+/* Dequeue one encode operations from ACC100 device in CB mode */
+static inline int
+dequeue_enc_one_op_cb(struct acc100_queue *q, struct rte_bbdev_enc_op **ref_op,
+ uint16_t total_dequeued_cbs, uint32_t *aq_dequeued)
+{
+ union acc100_dma_desc *desc, atom_desc;
+ union acc100_dma_rsp_desc rsp;
+ struct rte_bbdev_enc_op *op;
+ int i;
+
+ desc = q->ring_addr + ((q->sw_ring_tail + total_dequeued_cbs)
+ & q->sw_ring_wrap_mask);
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,
+ __ATOMIC_RELAXED);
+
+ /* Check fdone bit */
+ if (!(atom_desc.rsp.val & ACC100_FDONE))
+ return -1;
+
+ rsp.val = atom_desc.rsp.val;
+ rte_bbdev_log_debug("Resp. desc %p: %x", desc, rsp.val);
+
+ /* Dequeue */
+ op = desc->req.op_addr;
+
+ /* Clearing status, it will be set based on response */
+ op->status = 0;
+
+ op->status |= ((rsp.input_err)
+ ? (1 << RTE_BBDEV_DATA_ERROR) : 0);
+ op->status |= ((rsp.dma_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);
+ op->status |= ((rsp.fcw_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);
+
+ if (desc->req.last_desc_in_batch) {
+ (*aq_dequeued)++;
+ desc->req.last_desc_in_batch = 0;
+ }
+ desc->rsp.val = ACC100_DMA_DESC_TYPE;
+ desc->rsp.add_info_0 = 0; /*Reserved bits */
+ desc->rsp.add_info_1 = 0; /*Reserved bits */
+
+ /* Flag that the muxing cause loss of opaque data */
+ op->opaque_data = (void *)-1;
+ for (i = 0 ; i < desc->req.numCBs; i++)
+ ref_op[i] = op;
+
+ /* One CB (op) was successfully dequeued */
+ return desc->req.numCBs;
+}
+
+/* Dequeue one encode operations from ACC100 device in TB mode */
+static inline int
+dequeue_enc_one_op_tb(struct acc100_queue *q, struct rte_bbdev_enc_op **ref_op,
+ uint16_t total_dequeued_cbs, uint32_t *aq_dequeued)
+{
+ union acc100_dma_desc *desc, *last_desc, atom_desc;
+ union acc100_dma_rsp_desc rsp;
+ struct rte_bbdev_enc_op *op;
+ uint8_t i = 0;
+ uint16_t current_dequeued_cbs = 0, cbs_in_tb;
+
+ desc = q->ring_addr + ((q->sw_ring_tail + total_dequeued_cbs)
+ & q->sw_ring_wrap_mask);
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,
+ __ATOMIC_RELAXED);
+
+ /* Check fdone bit */
+ if (!(atom_desc.rsp.val & ACC100_FDONE))
+ return -1;
+
+ /* Get number of CBs in dequeued TB */
+ cbs_in_tb = desc->req.cbs_in_tb;
+ /* Get last CB */
+ last_desc = q->ring_addr + ((q->sw_ring_tail
+ + total_dequeued_cbs + cbs_in_tb - 1)
+ & q->sw_ring_wrap_mask);
+ /* Check if last CB in TB is ready to dequeue (and thus
+ * the whole TB) - checking sdone bit. If not return.
+ */
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)last_desc,
+ __ATOMIC_RELAXED);
+ if (!(atom_desc.rsp.val & ACC100_SDONE))
+ return -1;
+
+ /* Dequeue */
+ op = desc->req.op_addr;
+
+ /* Clearing status, it will be set based on response */
+ op->status = 0;
+
+ while (i < cbs_in_tb) {
+ desc = q->ring_addr + ((q->sw_ring_tail
+ + total_dequeued_cbs)
+ & q->sw_ring_wrap_mask);
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,
+ __ATOMIC_RELAXED);
+ rsp.val = atom_desc.rsp.val;
+ rte_bbdev_log_debug("Resp. desc %p: %x", desc,
+ rsp.val);
+
+ op->status |= ((rsp.input_err)
+ ? (1 << RTE_BBDEV_DATA_ERROR) : 0);
+ op->status |= ((rsp.dma_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);
+ op->status |= ((rsp.fcw_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);
+
+ if (desc->req.last_desc_in_batch) {
+ (*aq_dequeued)++;
+ desc->req.last_desc_in_batch = 0;
+ }
+ desc->rsp.val = ACC100_DMA_DESC_TYPE;
+ desc->rsp.add_info_0 = 0;
+ desc->rsp.add_info_1 = 0;
+ total_dequeued_cbs++;
+ current_dequeued_cbs++;
+ i++;
+ }
+
+ *ref_op = op;
+
+ return current_dequeued_cbs;
+}
+
+/* Dequeue one decode operation from ACC100 device in CB mode */
+static inline int
+dequeue_dec_one_op_cb(struct rte_bbdev_queue_data *q_data,
+ struct acc100_queue *q, struct rte_bbdev_dec_op **ref_op,
+ uint16_t dequeued_cbs, uint32_t *aq_dequeued)
+{
+ union acc100_dma_desc *desc, atom_desc;
+ union acc100_dma_rsp_desc rsp;
+ struct rte_bbdev_dec_op *op;
+
+ desc = q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)
+ & q->sw_ring_wrap_mask);
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,
+ __ATOMIC_RELAXED);
+
+ /* Check fdone bit */
+ if (!(atom_desc.rsp.val & ACC100_FDONE))
+ return -1;
+
+ rsp.val = atom_desc.rsp.val;
+ rte_bbdev_log_debug("Resp. desc %p: %x", desc, rsp.val);
+
+ /* Dequeue */
+ op = desc->req.op_addr;
+
+ /* Clearing status, it will be set based on response */
+ op->status = 0;
+ op->status |= ((rsp.input_err)
+ ? (1 << RTE_BBDEV_DATA_ERROR) : 0);
+ op->status |= ((rsp.dma_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);
+ op->status |= ((rsp.fcw_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);
+ if (op->status != 0) {
+ q_data->queue_stats.dequeue_err_count++;
+ acc100_check_ir(q->d);
+ }
+
+ /* CRC invalid if error exists */
+ if (!op->status)
+ op->status |= rsp.crc_status << RTE_BBDEV_CRC_ERROR;
+ op->turbo_dec.iter_count = (uint8_t) rsp.iter_cnt / 2;
+ /* Check if this is the last desc in batch (Atomic Queue) */
+ if (desc->req.last_desc_in_batch) {
+ (*aq_dequeued)++;
+ desc->req.last_desc_in_batch = 0;
+ }
+ desc->rsp.val = ACC100_DMA_DESC_TYPE;
+ desc->rsp.add_info_0 = 0;
+ desc->rsp.add_info_1 = 0;
+ *ref_op = op;
+
+ /* One CB (op) was successfully dequeued */
+ return 1;
+}
+
+/* Dequeue one decode operations from ACC100 device in CB mode */
+static inline int
+dequeue_ldpc_dec_one_op_cb(struct rte_bbdev_queue_data *q_data,
+ struct acc100_queue *q, struct rte_bbdev_dec_op **ref_op,
+ uint16_t dequeued_cbs, uint32_t *aq_dequeued)
+{
+ union acc100_dma_desc *desc, atom_desc;
+ union acc100_dma_rsp_desc rsp;
+ struct rte_bbdev_dec_op *op;
+
+ desc = q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)
+ & q->sw_ring_wrap_mask);
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,
+ __ATOMIC_RELAXED);
+
+ /* Check fdone bit */
+ if (!(atom_desc.rsp.val & ACC100_FDONE))
+ return -1;
+
+ rsp.val = atom_desc.rsp.val;
+
+ /* Dequeue */
+ op = desc->req.op_addr;
+
+ /* Clearing status, it will be set based on response */
+ op->status = 0;
+ op->status |= rsp.input_err << RTE_BBDEV_DATA_ERROR;
+ op->status |= rsp.dma_err << RTE_BBDEV_DRV_ERROR;
+ op->status |= rsp.fcw_err << RTE_BBDEV_DRV_ERROR;
+ if (op->status != 0)
+ q_data->queue_stats.dequeue_err_count++;
+
+ op->status |= rsp.crc_status << RTE_BBDEV_CRC_ERROR;
+ if (op->ldpc_dec.hard_output.length > 0 && !rsp.synd_ok)
+ op->status |= 1 << RTE_BBDEV_SYNDROME_ERROR;
+ op->ldpc_dec.iter_count = (uint8_t) rsp.iter_cnt;
+
+ if (op->status & (1 << RTE_BBDEV_DRV_ERROR))
+ acc100_check_ir(q->d);
+
+ /* Check if this is the last desc in batch (Atomic Queue) */
+ if (desc->req.last_desc_in_batch) {
+ (*aq_dequeued)++;
+ desc->req.last_desc_in_batch = 0;
+ }
+
+ desc->rsp.val = ACC100_DMA_DESC_TYPE;
+ desc->rsp.add_info_0 = 0;
+ desc->rsp.add_info_1 = 0;
+
+ *ref_op = op;
+
+ /* One CB (op) was successfully dequeued */
+ return 1;
+}
+
+/* Dequeue one decode operations from ACC100 device in TB mode. */
+static inline int
+dequeue_dec_one_op_tb(struct acc100_queue *q, struct rte_bbdev_dec_op **ref_op,
+ uint16_t dequeued_cbs, uint32_t *aq_dequeued)
+{
+ union acc100_dma_desc *desc, *last_desc, atom_desc;
+ union acc100_dma_rsp_desc rsp;
+ struct rte_bbdev_dec_op *op;
+ uint8_t cbs_in_tb = 1, cb_idx = 0;
+
+ desc = q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)
+ & q->sw_ring_wrap_mask);
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,
+ __ATOMIC_RELAXED);
+
+ /* Check fdone bit */
+ if (!(atom_desc.rsp.val & ACC100_FDONE))
+ return -1;
+
+ /* Dequeue */
+ op = desc->req.op_addr;
+
+ /* Get number of CBs in dequeued TB */
+ cbs_in_tb = desc->req.cbs_in_tb;
+ /* Get last CB */
+ last_desc = q->ring_addr + ((q->sw_ring_tail
+ + dequeued_cbs + cbs_in_tb - 1)
+ & q->sw_ring_wrap_mask);
+ /* Check if last CB in TB is ready to dequeue (and thus
+ * the whole TB) - checking sdone bit. If not return.
+ */
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)last_desc,
+ __ATOMIC_RELAXED);
+ if (!(atom_desc.rsp.val & ACC100_SDONE))
+ return -1;
+
+ /* Clearing status, it will be set based on response */
+ op->status = 0;
+
+ /* Read remaining CBs if exists */
+ while (cb_idx < cbs_in_tb) {
+ desc = q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)
+ & q->sw_ring_wrap_mask);
+ atom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,
+ __ATOMIC_RELAXED);
+ rsp.val = atom_desc.rsp.val;
+ rte_bbdev_log_debug("Resp. desc %p: %x", desc,
+ rsp.val);
+
+ op->status |= ((rsp.input_err)
+ ? (1 << RTE_BBDEV_DATA_ERROR) : 0);
+ op->status |= ((rsp.dma_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);
+ op->status |= ((rsp.fcw_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);
+
+ /* CRC invalid if error exists */
+ if (!op->status)
+ op->status |= rsp.crc_status << RTE_BBDEV_CRC_ERROR;
+ op->turbo_dec.iter_count = RTE_MAX((uint8_t) rsp.iter_cnt,
+ op->turbo_dec.iter_count);
+
+ /* Check if this is the last desc in batch (Atomic Queue) */
+ if (desc->req.last_desc_in_batch) {
+ (*aq_dequeued)++;
+ desc->req.last_desc_in_batch = 0;
+ }
+ desc->rsp.val = ACC100_DMA_DESC_TYPE;
+ desc->rsp.add_info_0 = 0;
+ desc->rsp.add_info_1 = 0;
+ dequeued_cbs++;
+ cb_idx++;
+ }
+
+ *ref_op = op;
+
+ return cb_idx;
+}
+
+/* Dequeue encode operations from ACC100 device. */
+static uint16_t
+acc100_dequeue_enc(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_enc_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ uint16_t dequeue_num;
+ uint32_t avail = q->sw_ring_head - q->sw_ring_tail;
+ uint32_t aq_dequeued = 0;
+ uint16_t i, dequeued_cbs = 0;
+ struct rte_bbdev_enc_op *op;
+ int ret;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (unlikely(ops == NULL || q == NULL)) {
+ rte_bbdev_log_debug("Unexpected undefined pointer");
+ return 0;
+ }
+#endif
+
+ dequeue_num = (avail < num) ? avail : num;
+
+ for (i = 0; i < dequeue_num; ++i) {
+ op = (q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)
+ & q->sw_ring_wrap_mask))->req.op_addr;
+ if (op->turbo_enc.code_block_mode == 0)
+ ret = dequeue_enc_one_op_tb(q, &ops[i], dequeued_cbs,
+ &aq_dequeued);
+ else
+ ret = dequeue_enc_one_op_cb(q, &ops[i], dequeued_cbs,
+ &aq_dequeued);
+
+ if (ret < 0)
+ break;
+ dequeued_cbs += ret;
+ }
+
+ q->aq_dequeued += aq_dequeued;
+ q->sw_ring_tail += dequeued_cbs;
+
+ /* Update enqueue stats */
+ q_data->queue_stats.dequeued_count += i;
+
+ return i;
+}
+
+/* Dequeue LDPC encode operations from ACC100 device. */
+static uint16_t
+acc100_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_enc_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ uint32_t avail = q->sw_ring_head - q->sw_ring_tail;
+ uint32_t aq_dequeued = 0;
+ uint16_t dequeue_num, i, dequeued_cbs = 0, dequeued_descs = 0;
+ int ret;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (unlikely(ops == 0 && q == NULL))
+ return 0;
+#endif
+
+ dequeue_num = RTE_MIN(avail, num);
+
+ for (i = 0; i < dequeue_num; i++) {
+ ret = dequeue_enc_one_op_cb(q, &ops[dequeued_cbs],
+ dequeued_descs, &aq_dequeued);
+ if (ret < 0)
+ break;
+ dequeued_cbs += ret;
+ dequeued_descs++;
+ if (dequeued_cbs >= num)
+ break;
+ }
+
+ q->aq_dequeued += aq_dequeued;
+ q->sw_ring_tail += dequeued_descs;
+
+ /* Update enqueue stats */
+ q_data->queue_stats.dequeued_count += dequeued_cbs;
+
+ return dequeued_cbs;
+}
+
+
+/* Dequeue decode operations from ACC100 device. */
+static uint16_t
+acc100_dequeue_dec(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_dec_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ uint16_t dequeue_num;
+ uint32_t avail = q->sw_ring_head - q->sw_ring_tail;
+ uint32_t aq_dequeued = 0;
+ uint16_t i;
+ uint16_t dequeued_cbs = 0;
+ struct rte_bbdev_dec_op *op;
+ int ret;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (unlikely(ops == 0 && q == NULL))
+ return 0;
+#endif
+
+ dequeue_num = (avail < num) ? avail : num;
+
+ for (i = 0; i < dequeue_num; ++i) {
+ op = (q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)
+ & q->sw_ring_wrap_mask))->req.op_addr;
+ if (op->turbo_dec.code_block_mode == 0)
+ ret = dequeue_dec_one_op_tb(q, &ops[i], dequeued_cbs,
+ &aq_dequeued);
+ else
+ ret = dequeue_dec_one_op_cb(q_data, q, &ops[i],
+ dequeued_cbs, &aq_dequeued);
+
+ if (ret < 0)
+ break;
+ dequeued_cbs += ret;
+ }
+
+ q->aq_dequeued += aq_dequeued;
+ q->sw_ring_tail += dequeued_cbs;
+
+ /* Update enqueue stats */
+ q_data->queue_stats.dequeued_count += i;
+
+ return i;
+}
+
+/* Dequeue decode operations from ACC100 device. */
+static uint16_t
+acc100_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
+ struct rte_bbdev_dec_op **ops, uint16_t num)
+{
+ struct acc100_queue *q = q_data->queue_private;
+ uint16_t dequeue_num;
+ uint32_t avail = q->sw_ring_head - q->sw_ring_tail;
+ uint32_t aq_dequeued = 0;
+ uint16_t i;
+ uint16_t dequeued_cbs = 0;
+ struct rte_bbdev_dec_op *op;
+ int ret;
+
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+ if (unlikely(ops == 0 && q == NULL))
+ return 0;
+#endif
+
+ dequeue_num = RTE_MIN(avail, num);
+
+ for (i = 0; i < dequeue_num; ++i) {
+ op = (q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)
+ & q->sw_ring_wrap_mask))->req.op_addr;
+ if (op->ldpc_dec.code_block_mode == 0)
+ ret = dequeue_dec_one_op_tb(q, &ops[i], dequeued_cbs,
+ &aq_dequeued);
+ else
+ ret = dequeue_ldpc_dec_one_op_cb(
+ q_data, q, &ops[i], dequeued_cbs,
+ &aq_dequeued);
+
+ if (ret < 0)
+ break;
+ dequeued_cbs += ret;
+ }
+
+ q->aq_dequeued += aq_dequeued;
+ q->sw_ring_tail += dequeued_cbs;
+
+ /* Update enqueue stats */
+ q_data->queue_stats.dequeued_count += i;
+
+ return i;
+}
+
+/* Initialization Function */
+static void
+acc100_bbdev_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
+{
+ struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
+
+ dev->dev_ops = &acc100_bbdev_ops;
+ dev->enqueue_enc_ops = acc100_enqueue_enc;
+ dev->enqueue_dec_ops = acc100_enqueue_dec;
+ dev->dequeue_enc_ops = acc100_dequeue_enc;
+ dev->dequeue_dec_ops = acc100_dequeue_dec;
+ dev->enqueue_ldpc_enc_ops = acc100_enqueue_ldpc_enc;
+ dev->enqueue_ldpc_dec_ops = acc100_enqueue_ldpc_dec;
+ dev->dequeue_ldpc_enc_ops = acc100_dequeue_ldpc_enc;
+ dev->dequeue_ldpc_dec_ops = acc100_dequeue_ldpc_dec;
+
+ ((struct acc100_device *) dev->data->dev_private)->pf_device =
+ !strcmp(drv->driver.name,
+ RTE_STR(ACC100PF_DRIVER_NAME));
+ ((struct acc100_device *) dev->data->dev_private)->mmio_base =
+ pci_dev->mem_resource[0].addr;
+
+ rte_bbdev_log_debug("Init device %s [%s] @ vaddr %p paddr %#"PRIx64"",
+ drv->driver.name, dev->data->name,
+ (void *)pci_dev->mem_resource[0].addr,
+ pci_dev->mem_resource[0].phys_addr);
+}
+
+static int acc100_pci_probe(struct rte_pci_driver *pci_drv,
+ struct rte_pci_device *pci_dev)
+{
+ struct rte_bbdev *bbdev = NULL;
+ char dev_name[RTE_BBDEV_NAME_MAX_LEN];
+
+ if (pci_dev == NULL) {
+ rte_bbdev_log(ERR, "NULL PCI device");
+ return -EINVAL;
+ }
+
+ rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
+
+ /* Allocate memory to be used privately by drivers */
+ bbdev = rte_bbdev_allocate(pci_dev->device.name);
+ if (bbdev == NULL)
+ return -ENODEV;
+
+ /* allocate device private memory */
+ bbdev->data->dev_private = rte_zmalloc_socket(dev_name,
+ sizeof(struct acc100_device), RTE_CACHE_LINE_SIZE,
+ pci_dev->device.numa_node);
+
+ if (bbdev->data->dev_private == NULL) {
+ rte_bbdev_log(CRIT,
+ "Allocate of %zu bytes for device \"%s\" failed",
+ sizeof(struct acc100_device), dev_name);
+ rte_bbdev_release(bbdev);
+ return -ENOMEM;
+ }
+
+ /* Fill HW specific part of device structure */
+ bbdev->device = &pci_dev->device;
+ bbdev->intr_handle = &pci_dev->intr_handle;
+ bbdev->data->socket_id = pci_dev->device.numa_node;
+
+ /* Invoke ACC100 device initialization function */
+ acc100_bbdev_init(bbdev, pci_drv);
+
+ rte_bbdev_log_debug("Initialised bbdev %s (id = %u)",
+ dev_name, bbdev->data->dev_id);
+ return 0;
+}
+
+static int acc100_pci_remove(struct rte_pci_device *pci_dev)
+{
+ struct rte_bbdev *bbdev;
+ int ret;
+ uint8_t dev_id;
+
+ if (pci_dev == NULL)
+ return -EINVAL;
+
+ /* Find device */
+ bbdev = rte_bbdev_get_named_dev(pci_dev->device.name);
+ if (bbdev == NULL) {
+ rte_bbdev_log(CRIT,
+ "Couldn't find HW dev \"%s\" to uninitialise it",
+ pci_dev->device.name);
+ return -ENODEV;
+ }
+ dev_id = bbdev->data->dev_id;
+
+ /* free device private memory before close */
+ rte_free(bbdev->data->dev_private);
+
+ /* Close device */
+ ret = rte_bbdev_close(dev_id);
+ if (ret < 0)
+ rte_bbdev_log(ERR,
+ "Device %i failed to close during uninit: %i",
+ dev_id, ret);
+
+ /* release bbdev from library */
+ rte_bbdev_release(bbdev);
+
+ rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id);
+
+ return 0;
+}
+
+static struct rte_pci_driver acc100_pci_pf_driver = {
+ .probe = acc100_pci_probe,
+ .remove = acc100_pci_remove,
+ .id_table = pci_id_acc100_pf_map,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING
+};