-/* CQE status. */
-enum mlx5_cqe_status {
- MLX5_CQE_STATUS_SW_OWN,
- MLX5_CQE_STATUS_HW_OWN,
- MLX5_CQE_STATUS_ERR,
-};
-
-/**
- * Check whether CQE is valid.
- *
- * @param cqe
- * Pointer to CQE.
- * @param cqes_n
- * Size of completion queue.
- * @param ci
- * Consumer index.
- *
- * @return
- * The CQE status.
- */
-static __rte_always_inline enum mlx5_cqe_status
-check_cqe(volatile struct mlx5_cqe *cqe, const uint16_t cqes_n,
- const uint16_t ci)
-{
- const uint16_t idx = ci & cqes_n;
- const uint8_t op_own = cqe->op_own;
- const uint8_t op_owner = MLX5_CQE_OWNER(op_own);
- const uint8_t op_code = MLX5_CQE_OPCODE(op_own);
-
- if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
- return MLX5_CQE_STATUS_HW_OWN;
- rte_cio_rmb();
- if (unlikely(op_code == MLX5_CQE_RESP_ERR ||
- op_code == MLX5_CQE_REQ_ERR))
- return MLX5_CQE_STATUS_ERR;
- return MLX5_CQE_STATUS_SW_OWN;
-}
-
-/**
- * Return the address of the WQE.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param wqe_ci
- * WQE consumer index.
- *
- * @return
- * WQE address.
- */
-static inline uintptr_t *
-tx_mlx5_wqe(struct mlx5_txq_data *txq, uint16_t ci)
-{
- ci &= ((1 << txq->wqe_n) - 1);
- return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
-}
-
-/**
- * Manage TX completions.
- *
- * When sending a burst, mlx5_tx_burst() posts several WRs.
- *
- * @param txq
- * Pointer to TX queue structure.
- */
-static __rte_always_inline void
-mlx5_tx_complete(struct mlx5_txq_data *txq)
-{
- const uint16_t elts_n = 1 << txq->elts_n;
- const uint16_t elts_m = elts_n - 1;
- const unsigned int cqe_n = 1 << txq->cqe_n;
- const unsigned int cqe_cnt = cqe_n - 1;
- uint16_t elts_free = txq->elts_tail;
- uint16_t elts_tail;
- uint16_t cq_ci = txq->cq_ci;
- volatile struct mlx5_cqe *cqe = NULL;
- volatile struct mlx5_wqe_ctrl *ctrl;
- struct rte_mbuf *m, *free[elts_n];
- struct rte_mempool *pool = NULL;
- unsigned int blk_n = 0;
-
- cqe = &(*txq->cqes)[cq_ci & cqe_cnt];
- if (unlikely(check_cqe(cqe, cqe_n, cq_ci)))
- return;
-#ifndef NDEBUG
- if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
- (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
- if (!check_cqe_seen(cqe)) {
- DRV_LOG(ERR, "unexpected error CQE, Tx stopped");
- rte_hexdump(stderr, "MLX5 TXQ:",
- (const void *)((uintptr_t)txq->wqes),
- ((1 << txq->wqe_n) *
- MLX5_WQE_SIZE));
- }
- return;
- }
-#endif /* NDEBUG */
- ++cq_ci;
- rte_cio_rmb();
- txq->wqe_pi = rte_be_to_cpu_16(cqe->wqe_counter);
- ctrl = (volatile struct mlx5_wqe_ctrl *)
- tx_mlx5_wqe(txq, txq->wqe_pi);
- elts_tail = ctrl->ctrl3;
- assert((elts_tail & elts_m) < (1 << txq->wqe_n));
- /* Free buffers. */
- while (elts_free != elts_tail) {
- m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
- if (likely(m != NULL)) {
- if (likely(m->pool == pool)) {
- free[blk_n++] = m;
- } else {
- if (likely(pool != NULL))
- rte_mempool_put_bulk(pool,
- (void *)free,
- blk_n);
- free[0] = m;
- pool = m->pool;
- blk_n = 1;
- }
- }
- }
- if (blk_n)
- rte_mempool_put_bulk(pool, (void *)free, blk_n);
-#ifndef NDEBUG
- elts_free = txq->elts_tail;
- /* Poisoning. */
- while (elts_free != elts_tail) {
- memset(&(*txq->elts)[elts_free & elts_m],
- 0x66,
- sizeof((*txq->elts)[elts_free & elts_m]));
- ++elts_free;
- }
-#endif
- txq->cq_ci = cq_ci;
- txq->elts_tail = elts_tail;
- /* Update the consumer index. */
- rte_compiler_barrier();
- *txq->cq_db = rte_cpu_to_be_32(cq_ci);
-}
-