1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2018 Intel Corporation
5 #include <rte_memcpy.h>
7 #include <rte_ethdev.h>
10 #include <rte_memzone.h>
11 #include <rte_errno.h>
12 #include <rte_string_fns.h>
13 #include <rte_pcapng.h>
15 #include "rte_pdump.h"
17 RTE_LOG_REGISTER_DEFAULT(pdump_logtype, NOTICE);
19 /* Macro for printing using RTE_LOG */
20 #define PDUMP_LOG(level, fmt, args...) \
21 rte_log(RTE_LOG_ ## level, pdump_logtype, "%s(): " fmt, \
24 /* Used for the multi-process communication */
25 #define PDUMP_MP "mp_pdump"
27 enum pdump_operation {
32 /* Internal version number in request */
34 V1 = 1, /* no filtering or snap */
38 struct pdump_request {
42 char device[RTE_DEV_NAME_MAX_LEN];
44 struct rte_ring *ring;
45 struct rte_mempool *mp;
47 const struct rte_bpf_prm *prm;
51 struct pdump_response {
57 static struct pdump_rxtx_cbs {
58 struct rte_ring *ring;
59 struct rte_mempool *mp;
60 const struct rte_eth_rxtx_callback *cb;
61 const struct rte_bpf *filter;
62 enum pdump_version ver;
64 } rx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT],
65 tx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
69 * The packet capture statistics keep track of packets
70 * accepted, filtered and dropped. These are per-queue
71 * and in memory between primary and secondary processes.
73 static const char MZ_RTE_PDUMP_STATS[] = "rte_pdump_stats";
75 struct rte_pdump_stats rx[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
76 struct rte_pdump_stats tx[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
77 const struct rte_memzone *mz;
80 /* Create a clone of mbuf to be placed into ring. */
82 pdump_copy(uint16_t port_id, uint16_t queue,
83 enum rte_pcapng_direction direction,
84 struct rte_mbuf **pkts, uint16_t nb_pkts,
85 const struct pdump_rxtx_cbs *cbs,
86 struct rte_pdump_stats *stats)
91 struct rte_mbuf *dup_bufs[nb_pkts];
93 struct rte_ring *ring;
94 struct rte_mempool *mp;
96 uint64_t rcs[nb_pkts];
99 rte_bpf_exec_burst(cbs->filter, (void **)pkts, rcs, nb_pkts);
101 ts = rte_get_tsc_cycles();
104 for (i = 0; i < nb_pkts; i++) {
106 * This uses same BPF return value convention as socket filter
107 * and pcap_offline_filter.
108 * if program returns zero
109 * then packet doesn't match the filter (will be ignored).
111 if (cbs->filter && rcs[i] == 0) {
112 __atomic_fetch_add(&stats->filtered,
113 1, __ATOMIC_RELAXED);
118 * If using pcapng then want to wrap packets
119 * otherwise a simple copy.
122 p = rte_pcapng_copy(port_id, queue,
123 pkts[i], mp, cbs->snaplen,
126 p = rte_pktmbuf_copy(pkts[i], mp, 0, cbs->snaplen);
128 if (unlikely(p == NULL))
129 __atomic_fetch_add(&stats->nombuf, 1, __ATOMIC_RELAXED);
131 dup_bufs[d_pkts++] = p;
134 __atomic_fetch_add(&stats->accepted, d_pkts, __ATOMIC_RELAXED);
136 ring_enq = rte_ring_enqueue_burst(ring, (void *)dup_bufs, d_pkts, NULL);
137 if (unlikely(ring_enq < d_pkts)) {
138 unsigned int drops = d_pkts - ring_enq;
140 __atomic_fetch_add(&stats->ringfull, drops, __ATOMIC_RELAXED);
141 rte_pktmbuf_free_bulk(&dup_bufs[ring_enq], drops);
146 pdump_rx(uint16_t port, uint16_t queue,
147 struct rte_mbuf **pkts, uint16_t nb_pkts,
148 uint16_t max_pkts __rte_unused, void *user_params)
150 const struct pdump_rxtx_cbs *cbs = user_params;
151 struct rte_pdump_stats *stats = &pdump_stats->rx[port][queue];
153 pdump_copy(port, queue, RTE_PCAPNG_DIRECTION_IN,
154 pkts, nb_pkts, cbs, stats);
159 pdump_tx(uint16_t port, uint16_t queue,
160 struct rte_mbuf **pkts, uint16_t nb_pkts, void *user_params)
162 const struct pdump_rxtx_cbs *cbs = user_params;
163 struct rte_pdump_stats *stats = &pdump_stats->tx[port][queue];
165 pdump_copy(port, queue, RTE_PCAPNG_DIRECTION_OUT,
166 pkts, nb_pkts, cbs, stats);
171 pdump_register_rx_callbacks(enum pdump_version ver,
172 uint16_t end_q, uint16_t port, uint16_t queue,
173 struct rte_ring *ring, struct rte_mempool *mp,
174 struct rte_bpf *filter,
175 uint16_t operation, uint32_t snaplen)
179 qid = (queue == RTE_PDUMP_ALL_QUEUES) ? 0 : queue;
180 for (; qid < end_q; qid++) {
181 struct pdump_rxtx_cbs *cbs = &rx_cbs[port][qid];
183 if (operation == ENABLE) {
186 "rx callback for port=%d queue=%d, already exists\n",
193 cbs->snaplen = snaplen;
194 cbs->filter = filter;
196 cbs->cb = rte_eth_add_first_rx_callback(port, qid,
198 if (cbs->cb == NULL) {
200 "failed to add rx callback, errno=%d\n",
204 } else if (operation == DISABLE) {
207 if (cbs->cb == NULL) {
209 "no existing rx callback for port=%d queue=%d\n",
213 ret = rte_eth_remove_rx_callback(port, qid, cbs->cb);
216 "failed to remove rx callback, errno=%d\n",
228 pdump_register_tx_callbacks(enum pdump_version ver,
229 uint16_t end_q, uint16_t port, uint16_t queue,
230 struct rte_ring *ring, struct rte_mempool *mp,
231 struct rte_bpf *filter,
232 uint16_t operation, uint32_t snaplen)
237 qid = (queue == RTE_PDUMP_ALL_QUEUES) ? 0 : queue;
238 for (; qid < end_q; qid++) {
239 struct pdump_rxtx_cbs *cbs = &tx_cbs[port][qid];
241 if (operation == ENABLE) {
244 "tx callback for port=%d queue=%d, already exists\n",
251 cbs->snaplen = snaplen;
252 cbs->filter = filter;
254 cbs->cb = rte_eth_add_tx_callback(port, qid, pdump_tx,
256 if (cbs->cb == NULL) {
258 "failed to add tx callback, errno=%d\n",
262 } else if (operation == DISABLE) {
265 if (cbs->cb == NULL) {
267 "no existing tx callback for port=%d queue=%d\n",
271 ret = rte_eth_remove_tx_callback(port, qid, cbs->cb);
274 "failed to remove tx callback, errno=%d\n",
286 set_pdump_rxtx_cbs(const struct pdump_request *p)
288 uint16_t nb_rx_q = 0, nb_tx_q = 0, end_q, queue;
291 struct rte_bpf *filter = NULL;
294 struct rte_ring *ring;
295 struct rte_mempool *mp;
297 /* Check for possible DPDK version mismatch */
298 if (!(p->ver == V1 || p->ver == V2)) {
300 "incorrect client version %u\n", p->ver);
305 if (p->prm->prog_arg.type != RTE_BPF_ARG_PTR_MBUF) {
307 "invalid BPF program type: %u\n",
308 p->prm->prog_arg.type);
312 filter = rte_bpf_load(p->prm);
313 if (filter == NULL) {
314 PDUMP_LOG(ERR, "cannot load BPF filter: %s\n",
315 rte_strerror(rte_errno));
326 ret = rte_eth_dev_get_port_by_name(p->device, &port);
329 "failed to get port id for device id=%s\n",
334 /* validation if packet capture is for all queues */
335 if (queue == RTE_PDUMP_ALL_QUEUES) {
336 struct rte_eth_dev_info dev_info;
338 ret = rte_eth_dev_info_get(port, &dev_info);
341 "Error during getting device (port %u) info: %s\n",
342 port, strerror(-ret));
346 nb_rx_q = dev_info.nb_rx_queues;
347 nb_tx_q = dev_info.nb_tx_queues;
348 if (nb_rx_q == 0 && flags & RTE_PDUMP_FLAG_RX) {
350 "number of rx queues cannot be 0\n");
353 if (nb_tx_q == 0 && flags & RTE_PDUMP_FLAG_TX) {
355 "number of tx queues cannot be 0\n");
358 if ((nb_tx_q == 0 || nb_rx_q == 0) &&
359 flags == RTE_PDUMP_FLAG_RXTX) {
361 "both tx&rx queues must be non zero\n");
366 /* register RX callback */
367 if (flags & RTE_PDUMP_FLAG_RX) {
368 end_q = (queue == RTE_PDUMP_ALL_QUEUES) ? nb_rx_q : queue + 1;
369 ret = pdump_register_rx_callbacks(p->ver, end_q, port, queue,
371 operation, p->snaplen);
376 /* register TX callback */
377 if (flags & RTE_PDUMP_FLAG_TX) {
378 end_q = (queue == RTE_PDUMP_ALL_QUEUES) ? nb_tx_q : queue + 1;
379 ret = pdump_register_tx_callbacks(p->ver, end_q, port, queue,
381 operation, p->snaplen);
390 pdump_server(const struct rte_mp_msg *mp_msg, const void *peer)
392 struct rte_mp_msg mp_resp;
393 const struct pdump_request *cli_req;
394 struct pdump_response *resp = (struct pdump_response *)&mp_resp.param;
396 /* recv client requests */
397 if (mp_msg->len_param != sizeof(*cli_req)) {
398 PDUMP_LOG(ERR, "failed to recv from client\n");
399 resp->err_value = -EINVAL;
401 cli_req = (const struct pdump_request *)mp_msg->param;
402 resp->ver = cli_req->ver;
403 resp->res_op = cli_req->op;
404 resp->err_value = set_pdump_rxtx_cbs(cli_req);
407 rte_strscpy(mp_resp.name, PDUMP_MP, RTE_MP_MAX_NAME_LEN);
408 mp_resp.len_param = sizeof(*resp);
410 if (rte_mp_reply(&mp_resp, peer) < 0) {
411 PDUMP_LOG(ERR, "failed to send to client:%s\n",
412 strerror(rte_errno));
422 const struct rte_memzone *mz;
425 mz = rte_memzone_reserve(MZ_RTE_PDUMP_STATS, sizeof(*pdump_stats),
428 PDUMP_LOG(ERR, "cannot allocate pdump statistics\n");
432 pdump_stats = mz->addr;
433 pdump_stats->mz = mz;
435 ret = rte_mp_action_register(PDUMP_MP, pdump_server);
436 if (ret && rte_errno != ENOTSUP)
442 rte_pdump_uninit(void)
444 rte_mp_action_unregister(PDUMP_MP);
446 if (pdump_stats != NULL) {
447 rte_memzone_free(pdump_stats->mz);
455 pdump_validate_ring_mp(struct rte_ring *ring, struct rte_mempool *mp)
457 if (ring == NULL || mp == NULL) {
458 PDUMP_LOG(ERR, "NULL ring or mempool\n");
462 if (mp->flags & RTE_MEMPOOL_F_SP_PUT ||
463 mp->flags & RTE_MEMPOOL_F_SC_GET) {
465 "mempool with SP or SC set not valid for pdump,"
466 "must have MP and MC set\n");
470 if (rte_ring_is_prod_single(ring) || rte_ring_is_cons_single(ring)) {
472 "ring with SP or SC set is not valid for pdump,"
473 "must have MP and MC set\n");
482 pdump_validate_flags(uint32_t flags)
484 if ((flags & RTE_PDUMP_FLAG_RXTX) == 0) {
486 "invalid flags, should be either rx/tx/rxtx\n");
491 /* mask off the flags we know about */
492 if (flags & ~(RTE_PDUMP_FLAG_RXTX | RTE_PDUMP_FLAG_PCAPNG)) {
494 "unknown flags: %#x\n", flags);
503 pdump_validate_port(uint16_t port, char *name)
507 if (port >= RTE_MAX_ETHPORTS) {
508 PDUMP_LOG(ERR, "Invalid port id %u\n", port);
513 ret = rte_eth_dev_get_name_by_port(port, name);
515 PDUMP_LOG(ERR, "port %u to name mapping failed\n",
525 pdump_prepare_client_request(const char *device, uint16_t queue,
526 uint32_t flags, uint32_t snaplen,
528 struct rte_ring *ring,
529 struct rte_mempool *mp,
530 const struct rte_bpf_prm *prm)
533 struct rte_mp_msg mp_req, *mp_rep;
534 struct rte_mp_reply mp_reply;
535 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
536 struct pdump_request *req = (struct pdump_request *)mp_req.param;
537 struct pdump_response *resp;
539 memset(req, 0, sizeof(*req));
541 req->ver = (flags & RTE_PDUMP_FLAG_PCAPNG) ? V2 : V1;
542 req->flags = flags & RTE_PDUMP_FLAG_RXTX;
545 rte_strscpy(req->device, device, sizeof(req->device));
547 if ((operation & ENABLE) != 0) {
551 req->snaplen = snaplen;
554 rte_strscpy(mp_req.name, PDUMP_MP, RTE_MP_MAX_NAME_LEN);
555 mp_req.len_param = sizeof(*req);
557 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0) {
558 mp_rep = &mp_reply.msgs[0];
559 resp = (struct pdump_response *)mp_rep->param;
560 rte_errno = resp->err_value;
561 if (!resp->err_value)
568 "client request for pdump enable/disable failed\n");
573 * There are two versions of this function, because although original API
574 * left place holder for future filter, it never checked the value.
575 * Therefore the API can't depend on application passing a non
579 pdump_enable(uint16_t port, uint16_t queue,
580 uint32_t flags, uint32_t snaplen,
581 struct rte_ring *ring, struct rte_mempool *mp,
582 const struct rte_bpf_prm *prm)
585 char name[RTE_DEV_NAME_MAX_LEN];
587 ret = pdump_validate_port(port, name);
590 ret = pdump_validate_ring_mp(ring, mp);
593 ret = pdump_validate_flags(flags);
598 snaplen = UINT32_MAX;
600 return pdump_prepare_client_request(name, queue, flags, snaplen,
601 ENABLE, ring, mp, prm);
605 rte_pdump_enable(uint16_t port, uint16_t queue, uint32_t flags,
606 struct rte_ring *ring,
607 struct rte_mempool *mp,
608 void *filter __rte_unused)
610 return pdump_enable(port, queue, flags, 0,
615 rte_pdump_enable_bpf(uint16_t port, uint16_t queue,
616 uint32_t flags, uint32_t snaplen,
617 struct rte_ring *ring,
618 struct rte_mempool *mp,
619 const struct rte_bpf_prm *prm)
621 return pdump_enable(port, queue, flags, snaplen,
626 pdump_enable_by_deviceid(const char *device_id, uint16_t queue,
627 uint32_t flags, uint32_t snaplen,
628 struct rte_ring *ring,
629 struct rte_mempool *mp,
630 const struct rte_bpf_prm *prm)
634 ret = pdump_validate_ring_mp(ring, mp);
637 ret = pdump_validate_flags(flags);
642 snaplen = UINT32_MAX;
644 return pdump_prepare_client_request(device_id, queue, flags, snaplen,
645 ENABLE, ring, mp, prm);
649 rte_pdump_enable_by_deviceid(char *device_id, uint16_t queue,
651 struct rte_ring *ring,
652 struct rte_mempool *mp,
653 void *filter __rte_unused)
655 return pdump_enable_by_deviceid(device_id, queue, flags, 0,
660 rte_pdump_enable_bpf_by_deviceid(const char *device_id, uint16_t queue,
661 uint32_t flags, uint32_t snaplen,
662 struct rte_ring *ring,
663 struct rte_mempool *mp,
664 const struct rte_bpf_prm *prm)
666 return pdump_enable_by_deviceid(device_id, queue, flags, snaplen,
671 rte_pdump_disable(uint16_t port, uint16_t queue, uint32_t flags)
674 char name[RTE_DEV_NAME_MAX_LEN];
676 ret = pdump_validate_port(port, name);
679 ret = pdump_validate_flags(flags);
683 ret = pdump_prepare_client_request(name, queue, flags, 0,
684 DISABLE, NULL, NULL, NULL);
690 rte_pdump_disable_by_deviceid(char *device_id, uint16_t queue,
695 ret = pdump_validate_flags(flags);
699 ret = pdump_prepare_client_request(device_id, queue, flags, 0,
700 DISABLE, NULL, NULL, NULL);
706 pdump_sum_stats(uint16_t port, uint16_t nq,
707 struct rte_pdump_stats stats[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT],
708 struct rte_pdump_stats *total)
710 uint64_t *sum = (uint64_t *)total;
715 for (qid = 0; qid < nq; qid++) {
716 const uint64_t *perq = (const uint64_t *)&stats[port][qid];
718 for (i = 0; i < sizeof(*total) / sizeof(uint64_t); i++) {
719 val = __atomic_load_n(&perq[i], __ATOMIC_RELAXED);
726 rte_pdump_stats(uint16_t port, struct rte_pdump_stats *stats)
728 struct rte_eth_dev_info dev_info;
729 const struct rte_memzone *mz;
732 memset(stats, 0, sizeof(*stats));
733 ret = rte_eth_dev_info_get(port, &dev_info);
736 "Error during getting device (port %u) info: %s\n",
737 port, strerror(-ret));
741 if (pdump_stats == NULL) {
742 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
743 /* rte_pdump_init was not called */
744 PDUMP_LOG(ERR, "pdump stats not initialized\n");
749 /* secondary process looks up the memzone */
750 mz = rte_memzone_lookup(MZ_RTE_PDUMP_STATS);
752 /* rte_pdump_init was not called in primary process?? */
753 PDUMP_LOG(ERR, "can not find pdump stats\n");
757 pdump_stats = mz->addr;
760 pdump_sum_stats(port, dev_info.nb_rx_queues, pdump_stats->rx, stats);
761 pdump_sum_stats(port, dev_info.nb_tx_queues, pdump_stats->tx, stats);