1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2018 Intel Corporation
6 #include <rte_ethdev.h>
9 #include <rte_memzone.h>
10 #include <rte_errno.h>
11 #include <rte_string_fns.h>
12 #include <rte_pcapng.h>
14 #include "rte_pdump.h"
16 RTE_LOG_REGISTER_DEFAULT(pdump_logtype, NOTICE);
18 /* Macro for printing using RTE_LOG */
19 #define PDUMP_LOG(level, fmt, args...) \
20 rte_log(RTE_LOG_ ## level, pdump_logtype, "%s(): " fmt, \
23 /* Used for the multi-process communication */
24 #define PDUMP_MP "mp_pdump"
26 enum pdump_operation {
31 /* Internal version number in request */
33 V1 = 1, /* no filtering or snap */
37 struct pdump_request {
41 char device[RTE_DEV_NAME_MAX_LEN];
43 struct rte_ring *ring;
44 struct rte_mempool *mp;
46 const struct rte_bpf_prm *prm;
50 struct pdump_response {
56 static struct pdump_rxtx_cbs {
57 struct rte_ring *ring;
58 struct rte_mempool *mp;
59 const struct rte_eth_rxtx_callback *cb;
60 const struct rte_bpf *filter;
61 enum pdump_version ver;
63 } rx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT],
64 tx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
68 * The packet capture statistics keep track of packets
69 * accepted, filtered and dropped. These are per-queue
70 * and in memory between primary and secondary processes.
72 static const char MZ_RTE_PDUMP_STATS[] = "rte_pdump_stats";
74 struct rte_pdump_stats rx[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
75 struct rte_pdump_stats tx[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
76 const struct rte_memzone *mz;
79 /* Create a clone of mbuf to be placed into ring. */
81 pdump_copy(uint16_t port_id, uint16_t queue,
82 enum rte_pcapng_direction direction,
83 struct rte_mbuf **pkts, uint16_t nb_pkts,
84 const struct pdump_rxtx_cbs *cbs,
85 struct rte_pdump_stats *stats)
90 struct rte_mbuf *dup_bufs[nb_pkts];
92 struct rte_ring *ring;
93 struct rte_mempool *mp;
95 uint64_t rcs[nb_pkts];
98 rte_bpf_exec_burst(cbs->filter, (void **)pkts, rcs, nb_pkts);
100 ts = rte_get_tsc_cycles();
103 for (i = 0; i < nb_pkts; i++) {
105 * This uses same BPF return value convention as socket filter
106 * and pcap_offline_filter.
107 * if program returns zero
108 * then packet doesn't match the filter (will be ignored).
110 if (cbs->filter && rcs[i] == 0) {
111 __atomic_fetch_add(&stats->filtered,
112 1, __ATOMIC_RELAXED);
117 * If using pcapng then want to wrap packets
118 * otherwise a simple copy.
121 p = rte_pcapng_copy(port_id, queue,
122 pkts[i], mp, cbs->snaplen,
125 p = rte_pktmbuf_copy(pkts[i], mp, 0, cbs->snaplen);
127 if (unlikely(p == NULL))
128 __atomic_fetch_add(&stats->nombuf, 1, __ATOMIC_RELAXED);
130 dup_bufs[d_pkts++] = p;
133 __atomic_fetch_add(&stats->accepted, d_pkts, __ATOMIC_RELAXED);
135 ring_enq = rte_ring_enqueue_burst(ring, (void *)dup_bufs, d_pkts, NULL);
136 if (unlikely(ring_enq < d_pkts)) {
137 unsigned int drops = d_pkts - ring_enq;
139 __atomic_fetch_add(&stats->ringfull, drops, __ATOMIC_RELAXED);
140 rte_pktmbuf_free_bulk(&dup_bufs[ring_enq], drops);
145 pdump_rx(uint16_t port, uint16_t queue,
146 struct rte_mbuf **pkts, uint16_t nb_pkts,
147 uint16_t max_pkts __rte_unused, void *user_params)
149 const struct pdump_rxtx_cbs *cbs = user_params;
150 struct rte_pdump_stats *stats = &pdump_stats->rx[port][queue];
152 pdump_copy(port, queue, RTE_PCAPNG_DIRECTION_IN,
153 pkts, nb_pkts, cbs, stats);
158 pdump_tx(uint16_t port, uint16_t queue,
159 struct rte_mbuf **pkts, uint16_t nb_pkts, void *user_params)
161 const struct pdump_rxtx_cbs *cbs = user_params;
162 struct rte_pdump_stats *stats = &pdump_stats->tx[port][queue];
164 pdump_copy(port, queue, RTE_PCAPNG_DIRECTION_OUT,
165 pkts, nb_pkts, cbs, stats);
170 pdump_register_rx_callbacks(enum pdump_version ver,
171 uint16_t end_q, uint16_t port, uint16_t queue,
172 struct rte_ring *ring, struct rte_mempool *mp,
173 struct rte_bpf *filter,
174 uint16_t operation, uint32_t snaplen)
178 qid = (queue == RTE_PDUMP_ALL_QUEUES) ? 0 : queue;
179 for (; qid < end_q; qid++) {
180 struct pdump_rxtx_cbs *cbs = &rx_cbs[port][qid];
182 if (operation == ENABLE) {
185 "rx callback for port=%d queue=%d, already exists\n",
192 cbs->snaplen = snaplen;
193 cbs->filter = filter;
195 cbs->cb = rte_eth_add_first_rx_callback(port, qid,
197 if (cbs->cb == NULL) {
199 "failed to add rx callback, errno=%d\n",
203 } else if (operation == DISABLE) {
206 if (cbs->cb == NULL) {
208 "no existing rx callback for port=%d queue=%d\n",
212 ret = rte_eth_remove_rx_callback(port, qid, cbs->cb);
215 "failed to remove rx callback, errno=%d\n",
227 pdump_register_tx_callbacks(enum pdump_version ver,
228 uint16_t end_q, uint16_t port, uint16_t queue,
229 struct rte_ring *ring, struct rte_mempool *mp,
230 struct rte_bpf *filter,
231 uint16_t operation, uint32_t snaplen)
236 qid = (queue == RTE_PDUMP_ALL_QUEUES) ? 0 : queue;
237 for (; qid < end_q; qid++) {
238 struct pdump_rxtx_cbs *cbs = &tx_cbs[port][qid];
240 if (operation == ENABLE) {
243 "tx callback for port=%d queue=%d, already exists\n",
250 cbs->snaplen = snaplen;
251 cbs->filter = filter;
253 cbs->cb = rte_eth_add_tx_callback(port, qid, pdump_tx,
255 if (cbs->cb == NULL) {
257 "failed to add tx callback, errno=%d\n",
261 } else if (operation == DISABLE) {
264 if (cbs->cb == NULL) {
266 "no existing tx callback for port=%d queue=%d\n",
270 ret = rte_eth_remove_tx_callback(port, qid, cbs->cb);
273 "failed to remove tx callback, errno=%d\n",
285 set_pdump_rxtx_cbs(const struct pdump_request *p)
287 uint16_t nb_rx_q = 0, nb_tx_q = 0, end_q, queue;
290 struct rte_bpf *filter = NULL;
293 struct rte_ring *ring;
294 struct rte_mempool *mp;
296 /* Check for possible DPDK version mismatch */
297 if (!(p->ver == V1 || p->ver == V2)) {
299 "incorrect client version %u\n", p->ver);
304 if (p->prm->prog_arg.type != RTE_BPF_ARG_PTR_MBUF) {
306 "invalid BPF program type: %u\n",
307 p->prm->prog_arg.type);
311 filter = rte_bpf_load(p->prm);
312 if (filter == NULL) {
313 PDUMP_LOG(ERR, "cannot load BPF filter: %s\n",
314 rte_strerror(rte_errno));
325 ret = rte_eth_dev_get_port_by_name(p->device, &port);
328 "failed to get port id for device id=%s\n",
333 /* validation if packet capture is for all queues */
334 if (queue == RTE_PDUMP_ALL_QUEUES) {
335 struct rte_eth_dev_info dev_info;
337 ret = rte_eth_dev_info_get(port, &dev_info);
340 "Error during getting device (port %u) info: %s\n",
341 port, strerror(-ret));
345 nb_rx_q = dev_info.nb_rx_queues;
346 nb_tx_q = dev_info.nb_tx_queues;
347 if (nb_rx_q == 0 && flags & RTE_PDUMP_FLAG_RX) {
349 "number of rx queues cannot be 0\n");
352 if (nb_tx_q == 0 && flags & RTE_PDUMP_FLAG_TX) {
354 "number of tx queues cannot be 0\n");
357 if ((nb_tx_q == 0 || nb_rx_q == 0) &&
358 flags == RTE_PDUMP_FLAG_RXTX) {
360 "both tx&rx queues must be non zero\n");
365 /* register RX callback */
366 if (flags & RTE_PDUMP_FLAG_RX) {
367 end_q = (queue == RTE_PDUMP_ALL_QUEUES) ? nb_rx_q : queue + 1;
368 ret = pdump_register_rx_callbacks(p->ver, end_q, port, queue,
370 operation, p->snaplen);
375 /* register TX callback */
376 if (flags & RTE_PDUMP_FLAG_TX) {
377 end_q = (queue == RTE_PDUMP_ALL_QUEUES) ? nb_tx_q : queue + 1;
378 ret = pdump_register_tx_callbacks(p->ver, end_q, port, queue,
380 operation, p->snaplen);
389 pdump_server(const struct rte_mp_msg *mp_msg, const void *peer)
391 struct rte_mp_msg mp_resp;
392 const struct pdump_request *cli_req;
393 struct pdump_response *resp = (struct pdump_response *)&mp_resp.param;
395 /* recv client requests */
396 if (mp_msg->len_param != sizeof(*cli_req)) {
397 PDUMP_LOG(ERR, "failed to recv from client\n");
398 resp->err_value = -EINVAL;
400 cli_req = (const struct pdump_request *)mp_msg->param;
401 resp->ver = cli_req->ver;
402 resp->res_op = cli_req->op;
403 resp->err_value = set_pdump_rxtx_cbs(cli_req);
406 rte_strscpy(mp_resp.name, PDUMP_MP, RTE_MP_MAX_NAME_LEN);
407 mp_resp.len_param = sizeof(*resp);
409 if (rte_mp_reply(&mp_resp, peer) < 0) {
410 PDUMP_LOG(ERR, "failed to send to client:%s\n",
411 strerror(rte_errno));
421 const struct rte_memzone *mz;
424 mz = rte_memzone_reserve(MZ_RTE_PDUMP_STATS, sizeof(*pdump_stats),
427 PDUMP_LOG(ERR, "cannot allocate pdump statistics\n");
431 pdump_stats = mz->addr;
432 pdump_stats->mz = mz;
434 ret = rte_mp_action_register(PDUMP_MP, pdump_server);
435 if (ret && rte_errno != ENOTSUP)
441 rte_pdump_uninit(void)
443 rte_mp_action_unregister(PDUMP_MP);
445 if (pdump_stats != NULL) {
446 rte_memzone_free(pdump_stats->mz);
454 pdump_validate_ring_mp(struct rte_ring *ring, struct rte_mempool *mp)
456 if (ring == NULL || mp == NULL) {
457 PDUMP_LOG(ERR, "NULL ring or mempool\n");
461 if (mp->flags & RTE_MEMPOOL_F_SP_PUT ||
462 mp->flags & RTE_MEMPOOL_F_SC_GET) {
464 "mempool with SP or SC set not valid for pdump,"
465 "must have MP and MC set\n");
469 if (rte_ring_is_prod_single(ring) || rte_ring_is_cons_single(ring)) {
471 "ring with SP or SC set is not valid for pdump,"
472 "must have MP and MC set\n");
481 pdump_validate_flags(uint32_t flags)
483 if ((flags & RTE_PDUMP_FLAG_RXTX) == 0) {
485 "invalid flags, should be either rx/tx/rxtx\n");
490 /* mask off the flags we know about */
491 if (flags & ~(RTE_PDUMP_FLAG_RXTX | RTE_PDUMP_FLAG_PCAPNG)) {
493 "unknown flags: %#x\n", flags);
502 pdump_validate_port(uint16_t port, char *name)
506 if (port >= RTE_MAX_ETHPORTS) {
507 PDUMP_LOG(ERR, "Invalid port id %u\n", port);
512 ret = rte_eth_dev_get_name_by_port(port, name);
514 PDUMP_LOG(ERR, "port %u to name mapping failed\n",
524 pdump_prepare_client_request(const char *device, uint16_t queue,
525 uint32_t flags, uint32_t snaplen,
527 struct rte_ring *ring,
528 struct rte_mempool *mp,
529 const struct rte_bpf_prm *prm)
532 struct rte_mp_msg mp_req, *mp_rep;
533 struct rte_mp_reply mp_reply;
534 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
535 struct pdump_request *req = (struct pdump_request *)mp_req.param;
536 struct pdump_response *resp;
538 memset(req, 0, sizeof(*req));
540 req->ver = (flags & RTE_PDUMP_FLAG_PCAPNG) ? V2 : V1;
541 req->flags = flags & RTE_PDUMP_FLAG_RXTX;
544 rte_strscpy(req->device, device, sizeof(req->device));
546 if ((operation & ENABLE) != 0) {
550 req->snaplen = snaplen;
553 rte_strscpy(mp_req.name, PDUMP_MP, RTE_MP_MAX_NAME_LEN);
554 mp_req.len_param = sizeof(*req);
556 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0) {
557 mp_rep = &mp_reply.msgs[0];
558 resp = (struct pdump_response *)mp_rep->param;
559 rte_errno = resp->err_value;
560 if (!resp->err_value)
567 "client request for pdump enable/disable failed\n");
572 * There are two versions of this function, because although original API
573 * left place holder for future filter, it never checked the value.
574 * Therefore the API can't depend on application passing a non
578 pdump_enable(uint16_t port, uint16_t queue,
579 uint32_t flags, uint32_t snaplen,
580 struct rte_ring *ring, struct rte_mempool *mp,
581 const struct rte_bpf_prm *prm)
584 char name[RTE_DEV_NAME_MAX_LEN];
586 ret = pdump_validate_port(port, name);
589 ret = pdump_validate_ring_mp(ring, mp);
592 ret = pdump_validate_flags(flags);
597 snaplen = UINT32_MAX;
599 return pdump_prepare_client_request(name, queue, flags, snaplen,
600 ENABLE, ring, mp, prm);
604 rte_pdump_enable(uint16_t port, uint16_t queue, uint32_t flags,
605 struct rte_ring *ring,
606 struct rte_mempool *mp,
607 void *filter __rte_unused)
609 return pdump_enable(port, queue, flags, 0,
614 rte_pdump_enable_bpf(uint16_t port, uint16_t queue,
615 uint32_t flags, uint32_t snaplen,
616 struct rte_ring *ring,
617 struct rte_mempool *mp,
618 const struct rte_bpf_prm *prm)
620 return pdump_enable(port, queue, flags, snaplen,
625 pdump_enable_by_deviceid(const char *device_id, uint16_t queue,
626 uint32_t flags, uint32_t snaplen,
627 struct rte_ring *ring,
628 struct rte_mempool *mp,
629 const struct rte_bpf_prm *prm)
633 ret = pdump_validate_ring_mp(ring, mp);
636 ret = pdump_validate_flags(flags);
641 snaplen = UINT32_MAX;
643 return pdump_prepare_client_request(device_id, queue, flags, snaplen,
644 ENABLE, ring, mp, prm);
648 rte_pdump_enable_by_deviceid(char *device_id, uint16_t queue,
650 struct rte_ring *ring,
651 struct rte_mempool *mp,
652 void *filter __rte_unused)
654 return pdump_enable_by_deviceid(device_id, queue, flags, 0,
659 rte_pdump_enable_bpf_by_deviceid(const char *device_id, uint16_t queue,
660 uint32_t flags, uint32_t snaplen,
661 struct rte_ring *ring,
662 struct rte_mempool *mp,
663 const struct rte_bpf_prm *prm)
665 return pdump_enable_by_deviceid(device_id, queue, flags, snaplen,
670 rte_pdump_disable(uint16_t port, uint16_t queue, uint32_t flags)
673 char name[RTE_DEV_NAME_MAX_LEN];
675 ret = pdump_validate_port(port, name);
678 ret = pdump_validate_flags(flags);
682 ret = pdump_prepare_client_request(name, queue, flags, 0,
683 DISABLE, NULL, NULL, NULL);
689 rte_pdump_disable_by_deviceid(char *device_id, uint16_t queue,
694 ret = pdump_validate_flags(flags);
698 ret = pdump_prepare_client_request(device_id, queue, flags, 0,
699 DISABLE, NULL, NULL, NULL);
705 pdump_sum_stats(uint16_t port, uint16_t nq,
706 struct rte_pdump_stats stats[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT],
707 struct rte_pdump_stats *total)
709 uint64_t *sum = (uint64_t *)total;
714 for (qid = 0; qid < nq; qid++) {
715 const uint64_t *perq = (const uint64_t *)&stats[port][qid];
717 for (i = 0; i < sizeof(*total) / sizeof(uint64_t); i++) {
718 val = __atomic_load_n(&perq[i], __ATOMIC_RELAXED);
725 rte_pdump_stats(uint16_t port, struct rte_pdump_stats *stats)
727 struct rte_eth_dev_info dev_info;
728 const struct rte_memzone *mz;
731 memset(stats, 0, sizeof(*stats));
732 ret = rte_eth_dev_info_get(port, &dev_info);
735 "Error during getting device (port %u) info: %s\n",
736 port, strerror(-ret));
740 if (pdump_stats == NULL) {
741 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
742 /* rte_pdump_init was not called */
743 PDUMP_LOG(ERR, "pdump stats not initialized\n");
748 /* secondary process looks up the memzone */
749 mz = rte_memzone_lookup(MZ_RTE_PDUMP_STATS);
751 /* rte_pdump_init was not called in primary process?? */
752 PDUMP_LOG(ERR, "can not find pdump stats\n");
756 pdump_stats = mz->addr;
759 pdump_sum_stats(port, dev_info.nb_rx_queues, pdump_stats->rx, stats);
760 pdump_sum_stats(port, dev_info.nb_tx_queues, pdump_stats->tx, stats);