1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
12 #include <rte_cycles.h>
13 #include <rte_ethdev.h>
14 #include <rte_metrics.h>
15 #include <rte_memzone.h>
16 #include <rte_lcore.h>
18 #include "rte_latencystats.h"
20 /** Nano seconds per second */
21 #define NS_PER_SEC 1E9
23 /** Clock cycles per nano second */
25 latencystat_cycles_per_ns(void)
27 return rte_get_timer_hz() / NS_PER_SEC;
30 /* Macros for printing using RTE_LOG */
31 #define RTE_LOGTYPE_LATENCY_STATS RTE_LOGTYPE_USER1
33 static const char *MZ_RTE_LATENCY_STATS = "rte_latencystats";
34 static int latency_stats_index;
35 static uint64_t samp_intvl;
36 static uint64_t timer_tsc;
37 static uint64_t prev_tsc;
39 struct rte_latency_stats {
40 float min_latency; /**< Minimum latency in nano seconds */
41 float avg_latency; /**< Average latency in nano seconds */
42 float max_latency; /**< Maximum latency in nano seconds */
43 float jitter; /** Latency variation */
46 static struct rte_latency_stats *glob_stats;
49 const struct rte_eth_rxtx_callback *cb;
52 static struct rxtx_cbs rx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
53 static struct rxtx_cbs tx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
55 struct latency_stats_nameoff {
56 char name[RTE_ETH_XSTATS_NAME_SIZE];
60 static const struct latency_stats_nameoff lat_stats_strings[] = {
61 {"min_latency_ns", offsetof(struct rte_latency_stats, min_latency)},
62 {"avg_latency_ns", offsetof(struct rte_latency_stats, avg_latency)},
63 {"max_latency_ns", offsetof(struct rte_latency_stats, max_latency)},
64 {"jitter_ns", offsetof(struct rte_latency_stats, jitter)},
67 #define NUM_LATENCY_STATS (sizeof(lat_stats_strings) / \
68 sizeof(lat_stats_strings[0]))
71 rte_latencystats_update(void)
74 float *stats_ptr = NULL;
75 uint64_t values[NUM_LATENCY_STATS] = {0};
78 for (i = 0; i < NUM_LATENCY_STATS; i++) {
79 stats_ptr = RTE_PTR_ADD(glob_stats,
80 lat_stats_strings[i].offset);
81 values[i] = (uint64_t)floor((*stats_ptr)/
82 latencystat_cycles_per_ns());
85 ret = rte_metrics_update_values(RTE_METRICS_GLOBAL,
87 values, NUM_LATENCY_STATS);
89 RTE_LOG(INFO, LATENCY_STATS, "Failed to push the stats\n");
95 rte_latencystats_fill_values(struct rte_metric_value *values)
98 float *stats_ptr = NULL;
100 for (i = 0; i < NUM_LATENCY_STATS; i++) {
101 stats_ptr = RTE_PTR_ADD(glob_stats,
102 lat_stats_strings[i].offset);
104 values[i].value = (uint64_t)floor((*stats_ptr)/
105 latencystat_cycles_per_ns());
110 add_time_stamps(uint16_t pid __rte_unused,
111 uint16_t qid __rte_unused,
112 struct rte_mbuf **pkts,
114 uint16_t max_pkts __rte_unused,
115 void *user_cb __rte_unused)
118 uint64_t diff_tsc, now;
121 * For every sample interval,
122 * time stamp is marked on one received packet.
125 for (i = 0; i < nb_pkts; i++) {
126 diff_tsc = now - prev_tsc;
127 timer_tsc += diff_tsc;
128 if (timer_tsc >= samp_intvl) {
129 pkts[i]->timestamp = now;
140 calc_latency(uint16_t pid __rte_unused,
141 uint16_t qid __rte_unused,
142 struct rte_mbuf **pkts,
144 void *_ __rte_unused)
146 unsigned int i, cnt = 0;
148 float latency[nb_pkts];
149 static float prev_latency;
151 * Alpha represents degree of weighting decrease in EWMA,
152 * a constant smoothing factor between 0 and 1. The value
153 * is used below for measuring average latency.
155 const float alpha = 0.2;
158 for (i = 0; i < nb_pkts; i++) {
159 if (pkts[i]->timestamp)
160 latency[cnt++] = now - pkts[i]->timestamp;
163 for (i = 0; i < cnt; i++) {
165 * The jitter is calculated as statistical mean of interpacket
166 * delay variation. The "jitter estimate" is computed by taking
167 * the absolute values of the ipdv sequence and applying an
168 * exponential filter with parameter 1/16 to generate the
169 * estimate. i.e J=J+(|D(i-1,i)|-J)/16. Where J is jitter,
170 * D(i-1,i) is difference in latency of two consecutive packets
172 * Reference: Calculated as per RFC 5481, sec 4.1,
173 * RFC 3393 sec 4.5, RFC 1889 sec.
175 glob_stats->jitter += (fabsf(prev_latency - latency[i])
176 - glob_stats->jitter)/16;
177 if (glob_stats->min_latency == 0)
178 glob_stats->min_latency = latency[i];
179 else if (latency[i] < glob_stats->min_latency)
180 glob_stats->min_latency = latency[i];
181 else if (latency[i] > glob_stats->max_latency)
182 glob_stats->max_latency = latency[i];
184 * The average latency is measured using exponential moving
185 * average, i.e. using EWMA
186 * https://en.wikipedia.org/wiki/Moving_average
188 glob_stats->avg_latency +=
189 alpha * (latency[i] - glob_stats->avg_latency);
190 prev_latency = latency[i];
197 rte_latencystats_init(uint64_t app_samp_intvl,
198 rte_latency_stats_flow_type_fn user_cb)
203 struct rxtx_cbs *cbs = NULL;
204 const char *ptr_strings[NUM_LATENCY_STATS] = {0};
205 const struct rte_memzone *mz = NULL;
206 const unsigned int flags = 0;
208 if (rte_memzone_lookup(MZ_RTE_LATENCY_STATS))
211 /** Allocate stats in shared memory fo multi process support */
212 mz = rte_memzone_reserve(MZ_RTE_LATENCY_STATS, sizeof(*glob_stats),
213 rte_socket_id(), flags);
215 RTE_LOG(ERR, LATENCY_STATS, "Cannot reserve memory: %s:%d\n",
220 glob_stats = mz->addr;
221 samp_intvl = app_samp_intvl * latencystat_cycles_per_ns();
223 /** Register latency stats with stats library */
224 for (i = 0; i < NUM_LATENCY_STATS; i++)
225 ptr_strings[i] = lat_stats_strings[i].name;
227 latency_stats_index = rte_metrics_reg_names(ptr_strings,
229 if (latency_stats_index < 0) {
230 RTE_LOG(DEBUG, LATENCY_STATS,
231 "Failed to register latency stats names\n");
235 /** Register Rx/Tx callbacks */
236 RTE_ETH_FOREACH_DEV(pid) {
237 struct rte_eth_dev_info dev_info;
238 rte_eth_dev_info_get(pid, &dev_info);
239 for (qid = 0; qid < dev_info.nb_rx_queues; qid++) {
240 cbs = &rx_cbs[pid][qid];
241 cbs->cb = rte_eth_add_first_rx_callback(pid, qid,
242 add_time_stamps, user_cb);
244 RTE_LOG(INFO, LATENCY_STATS, "Failed to "
245 "register Rx callback for pid=%d, "
246 "qid=%d\n", pid, qid);
248 for (qid = 0; qid < dev_info.nb_tx_queues; qid++) {
249 cbs = &tx_cbs[pid][qid];
250 cbs->cb = rte_eth_add_tx_callback(pid, qid,
251 calc_latency, user_cb);
253 RTE_LOG(INFO, LATENCY_STATS, "Failed to "
254 "register Tx callback for pid=%d, "
255 "qid=%d\n", pid, qid);
262 rte_latencystats_uninit(void)
267 struct rxtx_cbs *cbs = NULL;
269 /** De register Rx/Tx callbacks */
270 RTE_ETH_FOREACH_DEV(pid) {
271 struct rte_eth_dev_info dev_info;
272 rte_eth_dev_info_get(pid, &dev_info);
273 for (qid = 0; qid < dev_info.nb_rx_queues; qid++) {
274 cbs = &rx_cbs[pid][qid];
275 ret = rte_eth_remove_rx_callback(pid, qid, cbs->cb);
277 RTE_LOG(INFO, LATENCY_STATS, "failed to "
278 "remove Rx callback for pid=%d, "
279 "qid=%d\n", pid, qid);
281 for (qid = 0; qid < dev_info.nb_tx_queues; qid++) {
282 cbs = &tx_cbs[pid][qid];
283 ret = rte_eth_remove_tx_callback(pid, qid, cbs->cb);
285 RTE_LOG(INFO, LATENCY_STATS, "failed to "
286 "remove Tx callback for pid=%d, "
287 "qid=%d\n", pid, qid);
295 rte_latencystats_get_names(struct rte_metric_name *names, uint16_t size)
299 if (names == NULL || size < NUM_LATENCY_STATS)
300 return NUM_LATENCY_STATS;
302 for (i = 0; i < NUM_LATENCY_STATS; i++)
303 snprintf(names[i].name, sizeof(names[i].name),
304 "%s", lat_stats_strings[i].name);
306 return NUM_LATENCY_STATS;
310 rte_latencystats_get(struct rte_metric_value *values, uint16_t size)
312 if (size < NUM_LATENCY_STATS || values == NULL)
313 return NUM_LATENCY_STATS;
315 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
316 const struct rte_memzone *mz;
317 mz = rte_memzone_lookup(MZ_RTE_LATENCY_STATS);
319 RTE_LOG(ERR, LATENCY_STATS,
320 "Latency stats memzone not found\n");
323 glob_stats = mz->addr;
326 /* Retrieve latency stats */
327 rte_latencystats_fill_values(values);
329 return NUM_LATENCY_STATS;