1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2017 Intel Corporation
5 #include <rte_event_ring.h>
17 /* device instance specific */
20 sched_last_iter_bitmask,
21 sched_progress_last_iter,
28 poll_return, /* for zero-count and used also for port bucket loop */
31 /* qid port mapping specific */
33 pkts, /* note: qid-to-port pkts */
36 typedef uint64_t (*xstats_fn)(const struct sw_evdev *dev,
37 uint16_t obj_idx, /* port or queue id */
38 enum xstats_type stat, int extra_arg);
40 struct sw_xstats_entry {
41 struct rte_event_dev_xstats_name name;
44 enum xstats_type stat;
45 enum rte_event_dev_xstats_mode mode;
47 uint8_t reset_allowed; /* when set, this value can be reset */
48 uint64_t reset_value; /* an offset to be taken away to emulate resets */
52 get_dev_stat(const struct sw_evdev *sw, uint16_t obj_idx __rte_unused,
53 enum xstats_type type, int extra_arg __rte_unused)
56 case rx: return sw->stats.rx_pkts;
57 case tx: return sw->stats.tx_pkts;
58 case dropped: return sw->stats.rx_dropped;
59 case calls: return sw->sched_called;
60 case no_iq_enq: return sw->sched_no_iq_enqueues;
61 case no_cq_enq: return sw->sched_no_cq_enqueues;
62 case sched_last_iter_bitmask: return sw->sched_last_iter_bitmask;
63 case sched_progress_last_iter: return sw->sched_progress_last_iter;
70 get_port_stat(const struct sw_evdev *sw, uint16_t obj_idx,
71 enum xstats_type type, int extra_arg __rte_unused)
73 const struct sw_port *p = &sw->ports[obj_idx];
76 case rx: return p->stats.rx_pkts;
77 case tx: return p->stats.tx_pkts;
78 case dropped: return p->stats.rx_dropped;
79 case inflight: return p->inflights;
80 case pkt_cycles: return p->avg_pkt_ticks;
81 case calls: return p->total_polls;
82 case credits: return p->inflight_credits;
83 case poll_return: return p->zero_polls;
84 case rx_used: return rte_event_ring_count(p->rx_worker_ring);
85 case rx_free: return rte_event_ring_free_count(p->rx_worker_ring);
86 case tx_used: return rte_event_ring_count(p->cq_worker_ring);
87 case tx_free: return rte_event_ring_free_count(p->cq_worker_ring);
93 get_port_bucket_stat(const struct sw_evdev *sw, uint16_t obj_idx,
94 enum xstats_type type, int extra_arg)
96 const struct sw_port *p = &sw->ports[obj_idx];
99 case poll_return: return p->poll_buckets[extra_arg];
105 get_qid_stat(const struct sw_evdev *sw, uint16_t obj_idx,
106 enum xstats_type type, int extra_arg __rte_unused)
108 const struct sw_qid *qid = &sw->qids[obj_idx];
111 case rx: return qid->stats.rx_pkts;
112 case tx: return qid->stats.tx_pkts;
113 case dropped: return qid->stats.rx_dropped;
118 for (i = 0; i < RTE_DIM(qid->fids); i++)
119 infl += qid->fids[i].pcount;
128 get_qid_iq_stat(const struct sw_evdev *sw, uint16_t obj_idx,
129 enum xstats_type type, int extra_arg)
131 const struct sw_qid *qid = &sw->qids[obj_idx];
132 const int iq_idx = extra_arg;
135 case iq_used: return iq_count(&qid->iq[iq_idx]);
141 get_qid_port_stat(const struct sw_evdev *sw, uint16_t obj_idx,
142 enum xstats_type type, int extra_arg)
144 const struct sw_qid *qid = &sw->qids[obj_idx];
145 uint16_t port = extra_arg;
152 for (i = 0; i < RTE_DIM(qid->fids); i++)
153 if (qid->fids[i].cq == port)
159 return qid->to_port[port];
165 sw_xstats_init(struct sw_evdev *sw)
168 * define the stats names and types. Used to build up the device
170 * There are multiple set of stats:
173 * - per-port-dequeue-burst-sizes
178 * For each of these sets, we have three parallel arrays, one for the
179 * names, the other for the stat type parameter to be passed in the fn
180 * call to get that stat. The third array allows resetting or not.
181 * All these arrays must be kept in sync
183 static const char * const dev_stats[] = { "rx", "tx", "drop",
184 "sched_calls", "sched_no_iq_enq", "sched_no_cq_enq",
185 "sched_last_iter_bitmask", "sched_progress_last_iter",
187 static const enum xstats_type dev_types[] = { rx, tx, dropped,
188 calls, no_iq_enq, no_cq_enq, sched_last_iter_bitmask,
189 sched_progress_last_iter,
191 /* all device stats are allowed to be reset */
193 static const char * const port_stats[] = {"rx", "tx", "drop",
194 "inflight", "avg_pkt_cycles", "credits",
195 "rx_ring_used", "rx_ring_free",
196 "cq_ring_used", "cq_ring_free",
197 "dequeue_calls", "dequeues_returning_0",
199 static const enum xstats_type port_types[] = { rx, tx, dropped,
200 inflight, pkt_cycles, credits,
201 rx_used, rx_free, tx_used, tx_free,
204 static const uint8_t port_reset_allowed[] = {1, 1, 1,
210 static const char * const port_bucket_stats[] = {
211 "dequeues_returning" };
212 static const enum xstats_type port_bucket_types[] = { poll_return };
213 /* all bucket dequeues are allowed to be reset, handled in loop below */
215 static const char * const qid_stats[] = {"rx", "tx", "drop",
218 static const enum xstats_type qid_types[] = { rx, tx, dropped,
221 static const uint8_t qid_reset_allowed[] = {1, 1, 1,
225 static const char * const qid_iq_stats[] = { "used" };
226 static const enum xstats_type qid_iq_types[] = { iq_used };
229 static const char * const qid_port_stats[] = { "pinned_flows",
232 static const enum xstats_type qid_port_types[] = { pinned, pkts };
233 static const uint8_t qid_port_reset_allowed[] = {0, 1};
235 /* ---- end of stat definitions ---- */
237 /* check sizes, since a missed comma can lead to strings being
238 * joined by the compiler.
240 RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
241 RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
242 RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
243 RTE_BUILD_BUG_ON(RTE_DIM(qid_iq_stats) != RTE_DIM(qid_iq_types));
244 RTE_BUILD_BUG_ON(RTE_DIM(qid_port_stats) != RTE_DIM(qid_port_types));
245 RTE_BUILD_BUG_ON(RTE_DIM(port_bucket_stats) !=
246 RTE_DIM(port_bucket_types));
248 RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
249 RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
252 const uint32_t cons_bkt_shift =
253 (MAX_SW_CONS_Q_DEPTH >> SW_DEQ_STAT_BUCKET_SHIFT);
254 const unsigned int count = RTE_DIM(dev_stats) +
255 sw->port_count * RTE_DIM(port_stats) +
256 sw->port_count * RTE_DIM(port_bucket_stats) *
257 (cons_bkt_shift + 1) +
258 sw->qid_count * RTE_DIM(qid_stats) +
259 sw->qid_count * SW_IQS_MAX * RTE_DIM(qid_iq_stats) +
260 sw->qid_count * sw->port_count *
261 RTE_DIM(qid_port_stats);
262 unsigned int i, port, qid, iq, bkt, stat = 0;
264 sw->xstats = rte_zmalloc_socket(NULL, sizeof(sw->xstats[0]) * count, 0,
265 sw->data->socket_id);
266 if (sw->xstats == NULL)
269 #define sname sw->xstats[stat].name.name
270 for (i = 0; i < RTE_DIM(dev_stats); i++, stat++) {
271 sw->xstats[stat] = (struct sw_xstats_entry){
273 .stat = dev_types[i],
274 .mode = RTE_EVENT_DEV_XSTATS_DEVICE,
277 snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
279 sw->xstats_count_mode_dev = stat;
281 for (port = 0; port < sw->port_count; port++) {
282 sw->xstats_offset_for_port[port] = stat;
284 uint32_t count_offset = stat;
286 for (i = 0; i < RTE_DIM(port_stats); i++, stat++) {
287 sw->xstats[stat] = (struct sw_xstats_entry){
290 .stat = port_types[i],
291 .mode = RTE_EVENT_DEV_XSTATS_PORT,
292 .reset_allowed = port_reset_allowed[i],
294 snprintf(sname, sizeof(sname), "port_%u_%s",
295 port, port_stats[i]);
298 for (bkt = 0; bkt < (rte_event_ring_get_capacity(
299 sw->ports[port].cq_worker_ring) >>
300 SW_DEQ_STAT_BUCKET_SHIFT) + 1; bkt++) {
301 for (i = 0; i < RTE_DIM(port_bucket_stats); i++) {
302 sw->xstats[stat] = (struct sw_xstats_entry){
303 .fn = get_port_bucket_stat,
305 .stat = port_bucket_types[i],
306 .mode = RTE_EVENT_DEV_XSTATS_PORT,
310 snprintf(sname, sizeof(sname),
312 port, port_bucket_stats[i],
313 (bkt << SW_DEQ_STAT_BUCKET_SHIFT) + 1,
314 (bkt + 1) << SW_DEQ_STAT_BUCKET_SHIFT);
319 sw->xstats_count_per_port[port] = stat - count_offset;
322 sw->xstats_count_mode_port = stat - sw->xstats_count_mode_dev;
324 for (qid = 0; qid < sw->qid_count; qid++) {
325 uint32_t count_offset = stat;
326 sw->xstats_offset_for_qid[qid] = stat;
328 for (i = 0; i < RTE_DIM(qid_stats); i++, stat++) {
329 sw->xstats[stat] = (struct sw_xstats_entry){
332 .stat = qid_types[i],
333 .mode = RTE_EVENT_DEV_XSTATS_QUEUE,
334 .reset_allowed = qid_reset_allowed[i],
336 snprintf(sname, sizeof(sname), "qid_%u_%s",
339 for (iq = 0; iq < SW_IQS_MAX; iq++)
340 for (i = 0; i < RTE_DIM(qid_iq_stats); i++, stat++) {
341 sw->xstats[stat] = (struct sw_xstats_entry){
342 .fn = get_qid_iq_stat,
344 .stat = qid_iq_types[i],
345 .mode = RTE_EVENT_DEV_XSTATS_QUEUE,
349 snprintf(sname, sizeof(sname),
355 for (port = 0; port < sw->port_count; port++)
356 for (i = 0; i < RTE_DIM(qid_port_stats); i++, stat++) {
357 sw->xstats[stat] = (struct sw_xstats_entry){
358 .fn = get_qid_port_stat,
360 .stat = qid_port_types[i],
361 .mode = RTE_EVENT_DEV_XSTATS_QUEUE,
364 qid_port_reset_allowed[i],
366 snprintf(sname, sizeof(sname),
372 sw->xstats_count_per_qid[qid] = stat - count_offset;
375 sw->xstats_count_mode_queue = stat -
376 (sw->xstats_count_mode_dev + sw->xstats_count_mode_port);
379 sw->xstats_count = stat;
385 sw_xstats_uninit(struct sw_evdev *sw)
387 rte_free(sw->xstats);
388 sw->xstats_count = 0;
393 sw_xstats_get_names(const struct rte_eventdev *dev,
394 enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
395 struct rte_event_dev_xstats_name *xstats_names,
396 unsigned int *ids, unsigned int size)
398 const struct sw_evdev *sw = sw_pmd_priv_const(dev);
400 unsigned int xidx = 0;
402 uint32_t xstats_mode_count = 0;
403 uint32_t start_offset = 0;
406 case RTE_EVENT_DEV_XSTATS_DEVICE:
407 xstats_mode_count = sw->xstats_count_mode_dev;
409 case RTE_EVENT_DEV_XSTATS_PORT:
410 if (queue_port_id >= (signed int)sw->port_count)
412 xstats_mode_count = sw->xstats_count_per_port[queue_port_id];
413 start_offset = sw->xstats_offset_for_port[queue_port_id];
415 case RTE_EVENT_DEV_XSTATS_QUEUE:
416 if (queue_port_id >= (signed int)sw->qid_count)
418 xstats_mode_count = sw->xstats_count_per_qid[queue_port_id];
419 start_offset = sw->xstats_offset_for_qid[queue_port_id];
422 SW_LOG_ERR("Invalid mode received in sw_xstats_get_names()\n");
426 if (xstats_mode_count > size || !ids || !xstats_names)
427 return xstats_mode_count;
429 for (i = 0; i < sw->xstats_count && xidx < size; i++) {
430 if (sw->xstats[i].mode != mode)
433 if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
434 queue_port_id != sw->xstats[i].obj_idx)
437 xstats_names[xidx] = sw->xstats[i].name;
439 ids[xidx] = start_offset + xidx;
446 sw_xstats_update(struct sw_evdev *sw, enum rte_event_dev_xstats_mode mode,
447 uint8_t queue_port_id, const unsigned int ids[],
448 uint64_t values[], unsigned int n, const uint32_t reset,
449 const uint32_t ret_if_n_lt_nstats)
452 unsigned int xidx = 0;
454 RTE_SET_USED(queue_port_id);
456 uint32_t xstats_mode_count = 0;
459 case RTE_EVENT_DEV_XSTATS_DEVICE:
460 xstats_mode_count = sw->xstats_count_mode_dev;
462 case RTE_EVENT_DEV_XSTATS_PORT:
463 if (queue_port_id >= (signed int)sw->port_count)
465 xstats_mode_count = sw->xstats_count_per_port[queue_port_id];
467 case RTE_EVENT_DEV_XSTATS_QUEUE:
468 if (queue_port_id >= (signed int)sw->qid_count)
470 xstats_mode_count = sw->xstats_count_per_qid[queue_port_id];
473 SW_LOG_ERR("Invalid mode received in sw_xstats_get()\n");
477 /* this function can check num stats and return them (xstats_get() style
478 * behaviour) or ignore n for reset() of a single stat style behaviour.
480 if (ret_if_n_lt_nstats && xstats_mode_count > n)
481 return xstats_mode_count;
483 for (i = 0; i < n && xidx < xstats_mode_count; i++) {
484 struct sw_xstats_entry *xs = &sw->xstats[ids[i]];
485 if (ids[i] > sw->xstats_count || xs->mode != mode)
488 if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
489 queue_port_id != xs->obj_idx)
492 uint64_t val = xs->fn(sw, xs->obj_idx, xs->stat, xs->extra_arg)
498 if (xs->reset_allowed && reset)
499 xs->reset_value += val;
510 sw_xstats_get(const struct rte_eventdev *dev,
511 enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
512 const unsigned int ids[], uint64_t values[], unsigned int n)
514 struct sw_evdev *sw = sw_pmd_priv(dev);
515 const uint32_t reset = 0;
516 const uint32_t ret_n_lt_stats = 0;
517 return sw_xstats_update(sw, mode, queue_port_id, ids, values, n,
518 reset, ret_n_lt_stats);
522 sw_xstats_get_by_name(const struct rte_eventdev *dev,
523 const char *name, unsigned int *id)
525 const struct sw_evdev *sw = sw_pmd_priv_const(dev);
528 for (i = 0; i < sw->xstats_count; i++) {
529 struct sw_xstats_entry *xs = &sw->xstats[i];
530 if (strncmp(xs->name.name, name,
531 RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
534 return xs->fn(sw, xs->obj_idx, xs->stat, xs->extra_arg)
544 sw_xstats_reset_range(struct sw_evdev *sw, uint32_t start, uint32_t num)
547 for (i = start; i < start + num; i++) {
548 struct sw_xstats_entry *xs = &sw->xstats[i];
549 if (!xs->reset_allowed)
552 uint64_t val = xs->fn(sw, xs->obj_idx, xs->stat, xs->extra_arg);
553 xs->reset_value = val;
558 sw_xstats_reset_queue(struct sw_evdev *sw, uint8_t queue_id,
559 const uint32_t ids[], uint32_t nb_ids)
561 const uint32_t reset = 1;
562 const uint32_t ret_n_lt_stats = 0;
564 uint32_t nb_reset = sw_xstats_update(sw,
565 RTE_EVENT_DEV_XSTATS_QUEUE,
566 queue_id, ids, NULL, nb_ids,
567 reset, ret_n_lt_stats);
568 return nb_reset == nb_ids ? 0 : -EINVAL;
572 sw_xstats_reset_range(sw, sw->xstats_offset_for_qid[queue_id],
573 sw->xstats_count_per_qid[queue_id]);
579 sw_xstats_reset_port(struct sw_evdev *sw, uint8_t port_id,
580 const uint32_t ids[], uint32_t nb_ids)
582 const uint32_t reset = 1;
583 const uint32_t ret_n_lt_stats = 0;
584 int offset = sw->xstats_offset_for_port[port_id];
585 int nb_stat = sw->xstats_count_per_port[port_id];
588 uint32_t nb_reset = sw_xstats_update(sw,
589 RTE_EVENT_DEV_XSTATS_PORT, port_id,
591 reset, ret_n_lt_stats);
592 return nb_reset == nb_ids ? 0 : -EINVAL;
595 sw_xstats_reset_range(sw, offset, nb_stat);
600 sw_xstats_reset_dev(struct sw_evdev *sw, const uint32_t ids[], uint32_t nb_ids)
604 for (i = 0; i < nb_ids; i++) {
605 uint32_t id = ids[i];
606 if (id >= sw->xstats_count_mode_dev)
608 sw_xstats_reset_range(sw, id, 1);
611 for (i = 0; i < sw->xstats_count_mode_dev; i++)
612 sw_xstats_reset_range(sw, i, 1);
619 sw_xstats_reset(struct rte_eventdev *dev,
620 enum rte_event_dev_xstats_mode mode,
621 int16_t queue_port_id,
622 const uint32_t ids[],
625 struct sw_evdev *sw = sw_pmd_priv(dev);
628 /* handle -1 for queue_port_id here, looping over all ports/queues */
630 case RTE_EVENT_DEV_XSTATS_DEVICE:
631 sw_xstats_reset_dev(sw, ids, nb_ids);
633 case RTE_EVENT_DEV_XSTATS_PORT:
634 if (queue_port_id == -1) {
635 for (i = 0; i < sw->port_count; i++) {
636 err = sw_xstats_reset_port(sw, i, ids, nb_ids);
640 } else if (queue_port_id < (int16_t)sw->port_count)
641 sw_xstats_reset_port(sw, queue_port_id, ids, nb_ids);
643 case RTE_EVENT_DEV_XSTATS_QUEUE:
644 if (queue_port_id == -1) {
645 for (i = 0; i < sw->qid_count; i++) {
646 err = sw_xstats_reset_queue(sw, i, ids, nb_ids);
650 } else if (queue_port_id < (int16_t)sw->qid_count)
651 sw_xstats_reset_queue(sw, queue_port_id, ids, nb_ids);