+ claim_zero(pthread_mutex_unlock(&sh->txpp.mutex));
+}
+
+/*
+ * Read the current clock counter of an Ethernet device
+ *
+ * This returns the current raw clock value of an Ethernet device. It is
+ * a raw amount of ticks, with no given time reference.
+ * The value returned here is from the same clock than the one
+ * filling timestamp field of Rx/Tx packets when using hardware timestamp
+ * offload. Therefore it can be used to compute a precise conversion of
+ * the device clock to the real time.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param clock
+ * Pointer to the uint64_t that holds the raw clock value.
+ *
+ * @return
+ * - 0: Success.
+ * - -ENOTSUP: The function is not supported in this mode. Requires
+ * packet pacing module configured and started (tx_pp devarg)
+ */
+int
+mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_dev_ctx_shared *sh = priv->sh;
+ int ret;
+
+ if (sh->txpp.refcnt) {
+ struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue;
+ struct mlx5_cqe *cqe =
+ (struct mlx5_cqe *)(uintptr_t)wq->cq_obj.cqes;
+ union {
+ rte_int128_t u128;
+ struct mlx5_cqe_ts cts;
+ } to;
+ uint64_t ts;
+
+ mlx5_atomic_read_cqe((rte_int128_t *)&cqe->timestamp, &to.u128);
+ if (to.cts.op_own >> 4) {
+ DRV_LOG(DEBUG, "Clock Queue error sync lost.");
+ __atomic_fetch_add(&sh->txpp.err_clock_queue,
+ 1, __ATOMIC_RELAXED);
+ sh->txpp.sync_lost = 1;
+ return -EIO;
+ }
+ ts = rte_be_to_cpu_64(to.cts.timestamp);
+ ts = mlx5_txpp_convert_rx_ts(sh, ts);
+ *timestamp = ts;
+ return 0;
+ }
+ /* Not supported in isolated mode - kernel does not see the CQEs. */
+ if (priv->isolated || rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return -ENOTSUP;
+ ret = mlx5_read_clock(dev, timestamp);
+ return ret;
+}
+
+/**
+ * DPDK callback to clear device extended statistics.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success and stats is reset, negative errno value otherwise and
+ * rte_errno is set.
+ */
+int mlx5_txpp_xstats_reset(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_dev_ctx_shared *sh = priv->sh;
+
+ __atomic_store_n(&sh->txpp.err_miss_int, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&sh->txpp.err_rearm_queue, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&sh->txpp.err_clock_queue, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&sh->txpp.err_ts_past, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&sh->txpp.err_ts_future, 0, __ATOMIC_RELAXED);
+ return 0;
+}
+
+/**
+ * Routine to retrieve names of extended device statistics
+ * for packet send scheduling. It appends the specific stats names
+ * after the parts filled by preceding modules (eth stats, etc.)
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param[out] xstats_names
+ * Buffer to insert names into.
+ * @param n
+ * Number of names.
+ * @param n_used
+ * Number of names filled by preceding statistics modules.
+ *
+ * @return
+ * Number of xstats names.
+ */
+int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
+ struct rte_eth_xstat_name *xstats_names,
+ unsigned int n, unsigned int n_used)
+{
+ unsigned int n_txpp = RTE_DIM(mlx5_txpp_stat_names);
+ unsigned int i;
+
+ if (n >= n_used + n_txpp && xstats_names) {
+ for (i = 0; i < n_txpp; ++i) {
+ strncpy(xstats_names[i + n_used].name,
+ mlx5_txpp_stat_names[i],
+ RTE_ETH_XSTATS_NAME_SIZE);
+ xstats_names[i + n_used].name
+ [RTE_ETH_XSTATS_NAME_SIZE - 1] = 0;
+ }
+ }
+ return n_used + n_txpp;
+}
+
+static inline void
+mlx5_txpp_read_tsa(struct mlx5_dev_txpp *txpp,
+ struct mlx5_txpp_ts *tsa, uint16_t idx)
+{
+ do {
+ uint64_t ts, ci;
+
+ ts = __atomic_load_n(&txpp->tsa[idx].ts, __ATOMIC_RELAXED);
+ ci = __atomic_load_n(&txpp->tsa[idx].ci_ts, __ATOMIC_RELAXED);
+ rte_compiler_barrier();
+ if ((ci ^ ts) << MLX5_CQ_INDEX_WIDTH != 0)
+ continue;
+ if (__atomic_load_n(&txpp->tsa[idx].ts,
+ __ATOMIC_RELAXED) != ts)
+ continue;
+ if (__atomic_load_n(&txpp->tsa[idx].ci_ts,
+ __ATOMIC_RELAXED) != ci)
+ continue;
+ tsa->ts = ts;
+ tsa->ci_ts = ci;
+ return;
+ } while (true);
+}
+
+/*
+ * Jitter reflects the clock change between
+ * neighbours Clock Queue completions.
+ */
+static uint64_t
+mlx5_txpp_xstats_jitter(struct mlx5_dev_txpp *txpp)
+{
+ struct mlx5_txpp_ts tsa0, tsa1;
+ int64_t dts, dci;
+ uint16_t ts_p;
+
+ if (txpp->ts_n < 2) {
+ /* No gathered enough reports yet. */
+ return 0;
+ }
+ do {
+ int ts_0, ts_1;
+
+ ts_p = txpp->ts_p;
+ rte_compiler_barrier();
+ ts_0 = ts_p - 2;
+ if (ts_0 < 0)
+ ts_0 += MLX5_TXPP_REARM_SQ_SIZE;
+ ts_1 = ts_p - 1;
+ if (ts_1 < 0)
+ ts_1 += MLX5_TXPP_REARM_SQ_SIZE;
+ mlx5_txpp_read_tsa(txpp, &tsa0, ts_0);
+ mlx5_txpp_read_tsa(txpp, &tsa1, ts_1);
+ rte_compiler_barrier();
+ } while (ts_p != txpp->ts_p);
+ /* We have two neighbor reports, calculate the jitter. */
+ dts = tsa1.ts - tsa0.ts;
+ dci = (tsa1.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)) -
+ (tsa0.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH));
+ if (dci < 0)
+ dci += 1 << MLX5_CQ_INDEX_WIDTH;
+ dci *= txpp->tick;
+ return (dts > dci) ? dts - dci : dci - dts;
+}
+
+/*
+ * Wander reflects the long-term clock change
+ * over the entire length of all Clock Queue completions.
+ */
+static uint64_t
+mlx5_txpp_xstats_wander(struct mlx5_dev_txpp *txpp)
+{
+ struct mlx5_txpp_ts tsa0, tsa1;
+ int64_t dts, dci;
+ uint16_t ts_p;
+
+ if (txpp->ts_n < MLX5_TXPP_REARM_SQ_SIZE) {
+ /* No gathered enough reports yet. */
+ return 0;
+ }
+ do {
+ int ts_0, ts_1;
+
+ ts_p = txpp->ts_p;
+ rte_compiler_barrier();
+ ts_0 = ts_p - MLX5_TXPP_REARM_SQ_SIZE / 2 - 1;
+ if (ts_0 < 0)
+ ts_0 += MLX5_TXPP_REARM_SQ_SIZE;
+ ts_1 = ts_p - 1;
+ if (ts_1 < 0)
+ ts_1 += MLX5_TXPP_REARM_SQ_SIZE;
+ mlx5_txpp_read_tsa(txpp, &tsa0, ts_0);
+ mlx5_txpp_read_tsa(txpp, &tsa1, ts_1);
+ rte_compiler_barrier();
+ } while (ts_p != txpp->ts_p);
+ /* We have two neighbor reports, calculate the jitter. */
+ dts = tsa1.ts - tsa0.ts;
+ dci = (tsa1.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)) -
+ (tsa0.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH));
+ dci += 1 << MLX5_CQ_INDEX_WIDTH;
+ dci *= txpp->tick;
+ return (dts > dci) ? dts - dci : dci - dts;
+}
+
+/**
+ * Routine to retrieve extended device statistics
+ * for packet send scheduling. It appends the specific statistics
+ * after the parts filled by preceding modules (eth stats, etc.)
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] stats
+ * Pointer to rte extended stats table.
+ * @param n
+ * The size of the stats table.
+ * @param n_used
+ * Number of stats filled by preceding statistics modules.
+ *
+ * @return
+ * Number of extended stats on success and stats is filled,
+ * negative on error and rte_errno is set.
+ */
+int
+mlx5_txpp_xstats_get(struct rte_eth_dev *dev,
+ struct rte_eth_xstat *stats,
+ unsigned int n, unsigned int n_used)
+{
+ unsigned int n_txpp = RTE_DIM(mlx5_txpp_stat_names);
+
+ if (n >= n_used + n_txpp && stats) {
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_dev_ctx_shared *sh = priv->sh;
+ unsigned int i;
+
+ for (i = 0; i < n_txpp; ++i)
+ stats[n_used + i].id = n_used + i;
+ stats[n_used + 0].value =
+ __atomic_load_n(&sh->txpp.err_miss_int,
+ __ATOMIC_RELAXED);
+ stats[n_used + 1].value =
+ __atomic_load_n(&sh->txpp.err_rearm_queue,
+ __ATOMIC_RELAXED);
+ stats[n_used + 2].value =
+ __atomic_load_n(&sh->txpp.err_clock_queue,
+ __ATOMIC_RELAXED);
+ stats[n_used + 3].value =
+ __atomic_load_n(&sh->txpp.err_ts_past,
+ __ATOMIC_RELAXED);
+ stats[n_used + 4].value =
+ __atomic_load_n(&sh->txpp.err_ts_future,
+ __ATOMIC_RELAXED);
+ stats[n_used + 5].value = mlx5_txpp_xstats_jitter(&sh->txpp);
+ stats[n_used + 6].value = mlx5_txpp_xstats_wander(&sh->txpp);
+ stats[n_used + 7].value = sh->txpp.sync_lost;
+ }
+ return n_used + n_txpp;