- return 1; /* No CQE. */
-#ifndef NDEBUG
- if ((op_code == MLX5_CQE_RESP_ERR) ||
- (op_code == MLX5_CQE_REQ_ERR)) {
- volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
- uint8_t syndrome = err_cqe->syndrome;
-
- if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
- (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
- return 0;
- if (!check_cqe_seen(cqe)) {
- DRV_LOG(ERR,
- "unexpected CQE error %u (0x%02x) syndrome"
- " 0x%02x",
- op_code, op_code, syndrome);
- rte_hexdump(stderr, "MLX5 Error CQE:",
- (const void *)((uintptr_t)err_cqe),
- sizeof(*err_cqe));
- }
- return 1;
- } else if ((op_code != MLX5_CQE_RESP_SEND) &&
- (op_code != MLX5_CQE_REQ)) {
- if (!check_cqe_seen(cqe)) {
- DRV_LOG(ERR, "unexpected CQE opcode %u (0x%02x)",
- op_code, op_code);
- rte_hexdump(stderr, "MLX5 CQE:",
- (const void *)((uintptr_t)cqe),
- sizeof(*cqe));
- }
- return 1;
- }
-#endif /* NDEBUG */
- return 0;
-}
-
-/**
- * Return the address of the WQE.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param wqe_ci
- * WQE consumer index.
- *
- * @return
- * WQE address.
- */
-static inline uintptr_t *
-tx_mlx5_wqe(struct mlx5_txq_data *txq, uint16_t ci)
-{
- ci &= ((1 << txq->wqe_n) - 1);
- return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
-}
-
-/**
- * Manage TX completions.
- *
- * When sending a burst, mlx5_tx_burst() posts several WRs.
- *
- * @param txq
- * Pointer to TX queue structure.
- */
-static __rte_always_inline void
-mlx5_tx_complete(struct mlx5_txq_data *txq)
-{
- const uint16_t elts_n = 1 << txq->elts_n;
- const uint16_t elts_m = elts_n - 1;
- const unsigned int cqe_n = 1 << txq->cqe_n;
- const unsigned int cqe_cnt = cqe_n - 1;
- uint16_t elts_free = txq->elts_tail;
- uint16_t elts_tail;
- uint16_t cq_ci = txq->cq_ci;
- volatile struct mlx5_cqe *cqe = NULL;
- volatile struct mlx5_wqe_ctrl *ctrl;
- struct rte_mbuf *m, *free[elts_n];
- struct rte_mempool *pool = NULL;
- unsigned int blk_n = 0;
-
- cqe = &(*txq->cqes)[cq_ci & cqe_cnt];
- if (unlikely(check_cqe(cqe, cqe_n, cq_ci)))
- return;
-#ifndef NDEBUG
- if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
- (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
- if (!check_cqe_seen(cqe)) {
- DRV_LOG(ERR, "unexpected error CQE, Tx stopped");
- rte_hexdump(stderr, "MLX5 TXQ:",
- (const void *)((uintptr_t)txq->wqes),
- ((1 << txq->wqe_n) *
- MLX5_WQE_SIZE));
- }
- return;
- }
-#endif /* NDEBUG */
- ++cq_ci;
- txq->wqe_pi = rte_be_to_cpu_16(cqe->wqe_counter);
- ctrl = (volatile struct mlx5_wqe_ctrl *)
- tx_mlx5_wqe(txq, txq->wqe_pi);
- elts_tail = ctrl->ctrl3;
- assert((elts_tail & elts_m) < (1 << txq->wqe_n));
- /* Free buffers. */
- while (elts_free != elts_tail) {
- m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
- if (likely(m != NULL)) {
- if (likely(m->pool == pool)) {
- free[blk_n++] = m;
- } else {
- if (likely(pool != NULL))
- rte_mempool_put_bulk(pool,
- (void *)free,
- blk_n);
- free[0] = m;
- pool = m->pool;
- blk_n = 1;
- }
- }
- }
- if (blk_n)
- rte_mempool_put_bulk(pool, (void *)free, blk_n);
-#ifndef NDEBUG
- elts_free = txq->elts_tail;
- /* Poisoning. */
- while (elts_free != elts_tail) {
- memset(&(*txq->elts)[elts_free & elts_m],
- 0x66,
- sizeof((*txq->elts)[elts_free & elts_m]));
- ++elts_free;
- }
-#endif
- txq->cq_ci = cq_ci;
- txq->elts_tail = elts_tail;
- /* Update the consumer index. */
- rte_compiler_barrier();
- *txq->cq_db = rte_cpu_to_be_32(cq_ci);