#include <stdint.h>
#include <sys/queue.h>
-/* Verbs header. */
-/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
-#ifdef PEDANTIC
-#pragma GCC diagnostic ignored "-Wpedantic"
-#endif
-#include <infiniband/verbs.h>
-#include <infiniband/mlx5dv.h>
-#ifdef PEDANTIC
-#pragma GCC diagnostic error "-Wpedantic"
-#endif
-
#include <rte_mbuf.h>
#include <rte_mempool.h>
#include <rte_common.h>
#include <rte_io.h>
#include <rte_bus_pci.h>
#include <rte_malloc.h>
+#include <rte_cycles.h>
#include <mlx5_glue.h>
#include <mlx5_prm.h>
struct mlx5_rxq_data {
unsigned int csum:1; /* Enable checksum offloading. */
unsigned int hw_timestamp:1; /* Enable HW timestamp. */
+ unsigned int rt_timestamp:1; /* Realtime timestamp format. */
unsigned int vlan_strip:1; /* Enable VLAN stripping. */
unsigned int crc_present:1; /* CRC must be subtracted. */
unsigned int sges_n:3; /* Log 2 of SGEs (max buffers per packet). */
struct rte_mempool *mp;
struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */
struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish. */
+ struct mlx5_dev_ctx_shared *sh; /* Shared context. */
uint16_t idx; /* Queue index. */
struct mlx5_rxq_stats stats;
rte_xmm_t mbuf_initializer; /* Default rearm/flags for vectorized Rx. */
volatile uint32_t *cq_db; /* Completion queue doorbell. */
uint16_t port_id; /* Port ID of device. */
uint16_t idx; /* Queue index. */
+ uint64_t ts_mask; /* Timestamp flag dynamic mask. */
+ int32_t ts_offset; /* Timestamp field dynamic offset. */
+ struct mlx5_dev_ctx_shared *sh; /* Shared context. */
struct mlx5_txq_stats stats; /* TX queue counters. */
#ifndef RTE_ARCH_64
rte_spinlock_t *uar_lock;
enum mlx5_txq_obj_type {
MLX5_TXQ_OBJ_TYPE_IBV, /* mlx5_txq_obj with ibv_wq. */
+ MLX5_TXQ_OBJ_TYPE_DEVX_SQ, /* mlx5_txq_obj with mlx5_devx_sq. */
MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN,
/* mlx5_txq_obj with mlx5_devx_tq and hairpin support. */
};
/* DevX object for Sx queue. */
struct mlx5_devx_obj *tis; /* The TIS object. */
};
+ struct {
+ struct rte_eth_dev *dev;
+ struct mlx5_devx_obj *cq_devx;
+ struct mlx5dv_devx_umem *cq_umem;
+ void *cq_buf;
+ int64_t cq_dbrec_offset;
+ struct mlx5_devx_dbr_page *cq_dbrec_page;
+ struct mlx5_devx_obj *sq_devx;
+ struct mlx5dv_devx_umem *sq_umem;
+ void *sq_buf;
+ int64_t sq_dbrec_offset;
+ struct mlx5_devx_dbr_page *sq_dbrec_page;
+ };
};
};
void mlx5_hrxq_drop_release(struct rte_eth_dev *dev);
uint64_t mlx5_get_rx_port_offloads(void);
uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
+void mlx5_rxq_timestamp_set(struct rte_eth_dev *dev);
+
/* mlx5_txq.c */
void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
void txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl);
uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
+void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
/* mlx5_rxtx.c */
mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
}
+/**
+ * Convert timestamp from HW format to linear counter
+ * from Packet Pacing Clock Queue CQE timestamp format.
+ *
+ * @param sh
+ * Pointer to the device shared context. Might be needed
+ * to convert according current device configuration.
+ * @param ts
+ * Timestamp from CQE to convert.
+ * @return
+ * UTC in nanoseconds
+ */
+static __rte_always_inline uint64_t
+mlx5_txpp_convert_rx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t ts)
+{
+ RTE_SET_USED(sh);
+ return (ts & UINT32_MAX) + (ts >> 32) * NS_PER_S;
+}
+
+/**
+ * Convert timestamp from mbuf format to linear counter
+ * of Clock Queue completions (24 bits)
+ *
+ * @param sh
+ * Pointer to the device shared context to fetch Tx
+ * packet pacing timestamp and parameters.
+ * @param ts
+ * Timestamp from mbuf to convert.
+ * @return
+ * positive or zero value - completion ID to wait
+ * negative value - conversion error
+ */
+static __rte_always_inline int32_t
+mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts)
+{
+ uint64_t ts, ci;
+ uint32_t tick;
+
+ do {
+ /*
+ * Read atomically two uint64_t fields and compare lsb bits.
+ * It there is no match - the timestamp was updated in
+ * the service thread, data should be re-read.
+ */
+ rte_compiler_barrier();
+ ci = rte_atomic64_read(&sh->txpp.ts.ci_ts);
+ ts = rte_atomic64_read(&sh->txpp.ts.ts);
+ rte_compiler_barrier();
+ if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
+ break;
+ } while (true);
+ /* Perform the skew correction, positive value to send earlier. */
+ mts -= sh->txpp.skew;
+ mts -= ts;
+ if (unlikely(mts >= UINT64_MAX / 2)) {
+ /* We have negative integer, mts is in the past. */
+ rte_atomic32_inc(&sh->txpp.err_ts_past);
+ return -1;
+ }
+ tick = sh->txpp.tick;
+ MLX5_ASSERT(tick);
+ /* Convert delta to completions, round up. */
+ mts = (mts + tick - 1) / tick;
+ if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
+ /* We have mts is too distant future. */
+ rte_atomic32_inc(&sh->txpp.err_ts_future);
+ return -1;
+ }
+ mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
+ ci += mts;
+ ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
+ return ci;
+}
+
#endif /* RTE_PMD_MLX5_RXTX_H_ */
git.droids-corp.org / dpdk.git / blobdiff