-void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
- struct rte_eth_txq_info *qinfo);
-int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
- struct rte_eth_burst_mode *mode);
-
-/* mlx5_mr.c */
-
-void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
-uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb);
-uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
- struct rte_mempool *mp);
-int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova,
- size_t len);
-int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova,
- size_t len);
-
-/**
- * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
- * 64bit architectures.
- *
- * @param val
- * value to write in CPU endian format.
- * @param addr
- * Address to write to.
- * @param lock
- * Address of the lock to use for that UAR access.
- */
-static __rte_always_inline void
-__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
- rte_spinlock_t *lock __rte_unused)
-{
-#ifdef RTE_ARCH_64
- *(uint64_t *)addr = val;
-#else /* !RTE_ARCH_64 */
- rte_spinlock_lock(lock);
- *(uint32_t *)addr = val;
- rte_io_wmb();
- *((uint32_t *)addr + 1) = val >> 32;
- rte_spinlock_unlock(lock);
-#endif
-}
-
-/**
- * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
- * 64bit architectures while guaranteeing the order of execution with the
- * code being executed.
- *
- * @param val
- * value to write in CPU endian format.
- * @param addr
- * Address to write to.
- * @param lock
- * Address of the lock to use for that UAR access.
- */
-static __rte_always_inline void
-__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
-{
- rte_io_wmb();
- __mlx5_uar_write64_relaxed(val, addr, lock);
-}
-
-/* Assist macros, used instead of directly calling the functions they wrap. */
-#ifdef RTE_ARCH_64
-#define mlx5_uar_write64_relaxed(val, dst, lock) \
- __mlx5_uar_write64_relaxed(val, dst, NULL)
-#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL)
-#else
-#define mlx5_uar_write64_relaxed(val, dst, lock) \
- __mlx5_uar_write64_relaxed(val, dst, lock)
-#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
-#endif
-
-/**
- * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
- * cloned mbuf is allocated is returned instead.
- *
- * @param buf
- * Pointer to mbuf.
- *
- * @return
- * Memory pool where data is located for given mbuf.
- */
-static inline struct rte_mempool *
-mlx5_mb2mp(struct rte_mbuf *buf)
-{
- if (unlikely(RTE_MBUF_CLONED(buf)))
- return rte_mbuf_from_indirect(buf)->pool;
- return buf->pool;
-}
-
-/**
- * Query LKey from a packet buffer for Tx. If not found, add the mempool.
- *
- * @param txq
- * Pointer to Tx queue structure.
- * @param addr
- * Address to search.
- *
- * @return
- * Searched LKey on success, UINT32_MAX on no match.
- */
-static __rte_always_inline uint32_t
-mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
-{
- struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
- uintptr_t addr = (uintptr_t)mb->buf_addr;
- uint32_t lkey;
-
- /* Check generation bit to see if there's any change on existing MRs. */
- if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
- mlx5_mr_flush_local_cache(mr_ctrl);
- /* Linear search on MR cache array. */
- lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
- MLX5_MR_CACHE_N, addr);
- if (likely(lkey != UINT32_MAX))
- return lkey;
- /* Take slower bottom-half on miss. */
- return mlx5_tx_mb2mr_bh(txq, mb);
-}
-
-/**
- * Ring TX queue doorbell and flush the update if requested.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param wqe
- * Pointer to the last WQE posted in the NIC.
- * @param cond
- * Request for write memory barrier after BlueFlame update.
- */
-static __rte_always_inline void
-mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
- int cond)
-{
- uint64_t *dst = MLX5_TX_BFREG(txq);
- volatile uint64_t *src = ((volatile uint64_t *)wqe);
-
- rte_io_wmb();
- *txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
- /* Ensure ordering between DB record and BF copy. */
- rte_wmb();
- mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
- if (cond)
- rte_wmb();
-}
-
-/**
- * Ring TX queue doorbell and flush the update by write memory barrier.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param wqe
- * Pointer to the last WQE posted in the NIC.
- */
-static __rte_always_inline void
-mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
-{
- mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
-}
-
-/**
- * Convert timestamp from mbuf format to linear counter
- * of Clock Queue completions (24 bits)
- *
- * @param sh
- * Pointer to the device shared context to fetch Tx
- * packet pacing timestamp and parameters.
- * @param ts
- * Timestamp from mbuf to convert.
- * @return
- * positive or zero value - completion ID to wait
- * negative value - conversion error
- */
-static __rte_always_inline int32_t
-mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts)
-{
- uint64_t ts, ci;
- uint32_t tick;
-
- do {
- /*
- * Read atomically two uint64_t fields and compare lsb bits.
- * It there is no match - the timestamp was updated in
- * the service thread, data should be re-read.
- */
- rte_compiler_barrier();
- ci = __atomic_load_n(&sh->txpp.ts.ci_ts, __ATOMIC_RELAXED);
- ts = __atomic_load_n(&sh->txpp.ts.ts, __ATOMIC_RELAXED);
- rte_compiler_barrier();
- if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
- break;
- } while (true);
- /* Perform the skew correction, positive value to send earlier. */
- mts -= sh->txpp.skew;
- mts -= ts;
- if (unlikely(mts >= UINT64_MAX / 2)) {
- /* We have negative integer, mts is in the past. */
- __atomic_fetch_add(&sh->txpp.err_ts_past,
- 1, __ATOMIC_RELAXED);
- return -1;
- }
- tick = sh->txpp.tick;
- MLX5_ASSERT(tick);
- /* Convert delta to completions, round up. */
- mts = (mts + tick - 1) / tick;
- if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
- /* We have mts is too distant future. */
- __atomic_fetch_add(&sh->txpp.err_ts_future,
- 1, __ATOMIC_RELAXED);
- return -1;
- }
- mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
- ci += mts;
- ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
- return ci;
-}