1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
10 #include <sys/types.h>
11 #include <sys/queue.h>
18 #include <rte_common.h>
20 #include <rte_malloc.h>
21 #include <rte_memory.h>
22 #include <rte_memcpy.h>
24 #include <rte_launch.h>
25 #include <rte_atomic.h>
26 #include <rte_cycles.h>
27 #include <rte_prefetch.h>
28 #include <rte_lcore.h>
29 #include <rte_per_lcore.h>
30 #include <rte_branch_prediction.h>
31 #include <rte_interrupts.h>
32 #include <rte_random.h>
33 #include <rte_debug.h>
34 #include <rte_ether.h>
35 #include <rte_ethdev.h>
36 #include <rte_mempool.h>
39 #define RTE_LOGTYPE_LSI RTE_LOGTYPE_USER1
43 #define MAX_PKT_BURST 32
44 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
47 * Configurable number of RX/TX ring descriptors
49 #define RTE_TEST_RX_DESC_DEFAULT 1024
50 #define RTE_TEST_TX_DESC_DEFAULT 1024
51 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
52 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
54 /* ethernet addresses of ports */
55 static struct rte_ether_addr lsi_ports_eth_addr[RTE_MAX_ETHPORTS];
57 /* mask of enabled ports */
58 static uint32_t lsi_enabled_port_mask = 0;
60 static unsigned int lsi_rx_queue_per_lcore = 1;
62 /* destination port for L2 forwarding */
63 static unsigned lsi_dst_ports[RTE_MAX_ETHPORTS] = {0};
65 #define MAX_PKT_BURST 32
67 #define MAX_RX_QUEUE_PER_LCORE 16
68 #define MAX_TX_QUEUE_PER_PORT 16
69 /* List of queues must be polled for a give lcore. 8< */
70 struct lcore_queue_conf {
72 unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE];
74 } __rte_cache_aligned;
75 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
76 /* >8 End of list of queues to be polled. */
78 struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
80 /* Global configuration stored in a static structure. 8< */
81 static struct rte_eth_conf port_conf = {
86 .mq_mode = ETH_MQ_TX_NONE,
89 .lsc = 1, /**< lsc interrupt feature enabled */
92 /* >8 End of global configuration stored in a static structure. */
94 struct rte_mempool * lsi_pktmbuf_pool = NULL;
96 /* Per-port statistics struct */
97 struct lsi_port_statistics {
101 } __rte_cache_aligned;
102 struct lsi_port_statistics port_statistics[RTE_MAX_ETHPORTS];
104 /* A tsc-based timer responsible for triggering statistics printout */
105 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
106 #define MAX_TIMER_PERIOD 86400 /* 1 day max */
107 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; /* default period is 10 seconds */
109 /* Print out statistics on packets dropped */
113 struct rte_eth_link link;
114 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
117 total_packets_dropped = 0;
118 total_packets_tx = 0;
119 total_packets_rx = 0;
121 const char clr[] = { 27, '[', '2', 'J', '\0' };
122 const char topLeft[] = { 27, '[', '1', ';', '1', 'H','\0' };
125 /* Clear screen and move to top left */
126 printf("%s%s", clr, topLeft);
128 printf("\nPort statistics ====================================");
130 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
131 /* skip ports that are not enabled */
132 if ((lsi_enabled_port_mask & (1 << portid)) == 0)
135 memset(&link, 0, sizeof(link));
136 link_get_err = rte_eth_link_get_nowait(portid, &link);
137 printf("\nStatistics for port %u ------------------------------"
138 "\nLink status: %25s"
140 "\nLink duplex: %25s"
141 "\nPackets sent: %24"PRIu64
142 "\nPackets received: %20"PRIu64
143 "\nPackets dropped: %21"PRIu64,
145 link_get_err < 0 ? "Link get failed" :
146 (link.link_status ? "Link up" : "Link down"),
147 link_get_err < 0 ? "0" :
148 rte_eth_link_speed_to_str(link.link_speed),
149 link_get_err < 0 ? "Link get failed" :
150 (link.link_duplex == ETH_LINK_FULL_DUPLEX ? \
151 "full-duplex" : "half-duplex"),
152 port_statistics[portid].tx,
153 port_statistics[portid].rx,
154 port_statistics[portid].dropped);
156 total_packets_dropped += port_statistics[portid].dropped;
157 total_packets_tx += port_statistics[portid].tx;
158 total_packets_rx += port_statistics[portid].rx;
160 printf("\nAggregate statistics ==============================="
161 "\nTotal packets sent: %18"PRIu64
162 "\nTotal packets received: %14"PRIu64
163 "\nTotal packets dropped: %15"PRIu64,
166 total_packets_dropped);
167 printf("\n====================================================\n");
172 /* Replacing the source and destination MAC addresses. 8< */
174 lsi_simple_forward(struct rte_mbuf *m, unsigned portid)
176 struct rte_ether_hdr *eth;
178 unsigned dst_port = lsi_dst_ports[portid];
180 struct rte_eth_dev_tx_buffer *buffer;
182 eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
184 /* 02:00:00:00:00:xx */
185 tmp = ð->d_addr.addr_bytes[0];
186 *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40);
189 rte_ether_addr_copy(&lsi_ports_eth_addr[dst_port], ð->s_addr);
191 buffer = tx_buffer[dst_port];
192 sent = rte_eth_tx_buffer(dst_port, 0, buffer, m);
194 port_statistics[dst_port].tx += sent;
196 /* >8 End of replacing the source and destination MAC addresses. */
198 /* main processing loop */
202 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
206 uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc;
207 unsigned i, j, portid, nb_rx;
208 struct lcore_queue_conf *qconf;
209 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S *
211 struct rte_eth_dev_tx_buffer *buffer;
216 lcore_id = rte_lcore_id();
217 qconf = &lcore_queue_conf[lcore_id];
219 if (qconf->n_rx_port == 0) {
220 RTE_LOG(INFO, LSI, "lcore %u has nothing to do\n", lcore_id);
224 RTE_LOG(INFO, LSI, "entering main loop on lcore %u\n", lcore_id);
226 for (i = 0; i < qconf->n_rx_port; i++) {
228 portid = qconf->rx_port_list[i];
229 RTE_LOG(INFO, LSI, " -- lcoreid=%u portid=%u\n", lcore_id,
235 /* Draining TX queue in its main loop. 8< */
236 cur_tsc = rte_rdtsc();
239 * TX burst queue drain
241 diff_tsc = cur_tsc - prev_tsc;
242 if (unlikely(diff_tsc > drain_tsc)) {
244 for (i = 0; i < qconf->n_rx_port; i++) {
246 portid = lsi_dst_ports[qconf->rx_port_list[i]];
247 buffer = tx_buffer[portid];
249 sent = rte_eth_tx_buffer_flush(portid, 0, buffer);
251 port_statistics[portid].tx += sent;
255 /* if timer is enabled */
256 if (timer_period > 0) {
258 /* advance the timer */
259 timer_tsc += diff_tsc;
261 /* if timer has reached its timeout */
262 if (unlikely(timer_tsc >= (uint64_t) timer_period)) {
264 /* do this only on main core */
265 if (lcore_id == rte_get_main_lcore()) {
267 /* reset the timer */
275 /* >8 End of draining TX queue in its main loop. */
277 /* Read packet from RX queues. 8< */
278 for (i = 0; i < qconf->n_rx_port; i++) {
280 portid = qconf->rx_port_list[i];
281 nb_rx = rte_eth_rx_burst((uint8_t) portid, 0,
282 pkts_burst, MAX_PKT_BURST);
284 port_statistics[portid].rx += nb_rx;
286 for (j = 0; j < nb_rx; j++) {
288 rte_prefetch0(rte_pktmbuf_mtod(m, void *));
289 lsi_simple_forward(m, portid);
292 /* >8 End of reading packet from RX queues. */
297 lsi_launch_one_lcore(__rte_unused void *dummy)
305 lsi_usage(const char *prgname)
307 printf("%s [EAL options] -- -p PORTMASK [-q NQ]\n"
308 " -p PORTMASK: hexadecimal bitmask of ports to configure\n"
309 " -q NQ: number of queue (=ports) per lcore (default is 1)\n"
310 " -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n",
315 lsi_parse_portmask(const char *portmask)
320 /* parse hexadecimal string */
321 pm = strtoul(portmask, &end, 16);
322 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
329 lsi_parse_nqueue(const char *q_arg)
334 /* parse hexadecimal string */
335 n = strtoul(q_arg, &end, 10);
336 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
340 if (n >= MAX_RX_QUEUE_PER_LCORE)
347 lsi_parse_timer_period(const char *q_arg)
352 /* parse number string */
353 n = strtol(q_arg, &end, 10);
354 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
356 if (n >= MAX_TIMER_PERIOD)
362 /* Parse the argument given in the command line of the application */
364 lsi_parse_args(int argc, char **argv)
369 char *prgname = argv[0];
370 static struct option lgopts[] = {
376 while ((opt = getopt_long(argc, argvopt, "p:q:T:",
377 lgopts, &option_index)) != EOF) {
382 lsi_enabled_port_mask = lsi_parse_portmask(optarg);
383 if (lsi_enabled_port_mask == 0) {
384 printf("invalid portmask\n");
392 lsi_rx_queue_per_lcore = lsi_parse_nqueue(optarg);
393 if (lsi_rx_queue_per_lcore == 0) {
394 printf("invalid queue number\n");
402 timer_period = lsi_parse_timer_period(optarg) * 1000 * TIMER_MILLISECOND;
403 if (timer_period < 0) {
404 printf("invalid timer period\n");
422 argv[optind-1] = prgname;
425 optind = 1; /* reset getopt lib */
430 * It will be called as the callback for specified port after a LSI interrupt
431 * has been fully handled. This callback needs to be implemented carefully as
432 * it will be called in the interrupt host thread which is different from the
433 * application main thread.
440 * Pointer to(address of) the parameters.
446 /* lsi_event_callback 8< */
448 lsi_event_callback(uint16_t port_id, enum rte_eth_event_type type, void *param,
451 struct rte_eth_link link;
453 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN];
456 RTE_SET_USED(ret_param);
458 printf("\n\nIn registered callback...\n");
459 printf("Event type: %s\n", type == RTE_ETH_EVENT_INTR_LSC ? "LSC interrupt" : "unknown event");
460 ret = rte_eth_link_get_nowait(port_id, &link);
462 printf("Failed link get on port %d: %s\n",
463 port_id, rte_strerror(-ret));
466 rte_eth_link_to_str(link_status_text, sizeof(link_status_text), &link);
467 printf("Port %d %s\n\n", port_id, link_status_text);
471 /* >8 End of registering one or more callbacks. */
473 /* Check the link status of all ports in up to 9s, and print them finally */
475 check_all_ports_link_status(uint16_t port_num, uint32_t port_mask)
477 #define CHECK_INTERVAL 100 /* 100ms */
478 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
479 uint8_t count, all_ports_up, print_flag = 0;
481 struct rte_eth_link link;
483 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN];
485 printf("\nChecking link status");
487 for (count = 0; count <= MAX_CHECK_TIME; count++) {
489 for (portid = 0; portid < port_num; portid++) {
490 if ((port_mask & (1 << portid)) == 0)
492 memset(&link, 0, sizeof(link));
493 ret = rte_eth_link_get_nowait(portid, &link);
497 printf("Port %u link get failed: %s\n",
498 portid, rte_strerror(-ret));
501 /* print link status if flag set */
502 if (print_flag == 1) {
503 rte_eth_link_to_str(link_status_text,
504 sizeof(link_status_text), &link);
505 printf("Port %d %s", portid,
509 /* clear all_ports_up flag if any link down */
510 if (link.link_status == ETH_LINK_DOWN) {
515 /* after finally printing all link status, get out */
519 if (all_ports_up == 0) {
522 rte_delay_ms(CHECK_INTERVAL);
525 /* set the print_flag if all ports up or timeout */
526 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
534 main(int argc, char **argv)
536 struct lcore_queue_conf *qconf;
539 uint16_t portid, portid_last = 0;
540 unsigned lcore_id, rx_lcore_id;
541 unsigned nb_ports_in_mask = 0;
544 ret = rte_eal_init(argc, argv);
546 rte_exit(EXIT_FAILURE, "rte_eal_init failed");
550 /* parse application arguments (after the EAL ones) */
551 ret = lsi_parse_args(argc, argv);
553 rte_exit(EXIT_FAILURE, "Invalid arguments");
555 /* create the mbuf pool */
557 rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 32, 0,
558 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
559 if (lsi_pktmbuf_pool == NULL)
560 rte_panic("Cannot init mbuf pool\n");
562 nb_ports = rte_eth_dev_count_avail();
564 rte_panic("No Ethernet port - bye\n");
566 /* Each logical core is assigned a dedicated TX queue on each port. 8< */
567 for (portid = 0; portid < nb_ports; portid++) {
568 /* skip ports that are not enabled */
569 if ((lsi_enabled_port_mask & (1 << portid)) == 0)
572 /* save the destination port id */
573 if (nb_ports_in_mask % 2) {
574 lsi_dst_ports[portid] = portid_last;
575 lsi_dst_ports[portid_last] = portid;
578 portid_last = portid;
582 /* >8 End of assigning logical core. */
583 if (nb_ports_in_mask < 2 || nb_ports_in_mask % 2)
584 rte_exit(EXIT_FAILURE, "Current enabled port number is %u, "
585 "but it should be even and at least 2\n",
589 qconf = &lcore_queue_conf[rx_lcore_id];
591 /* Initialize the port/queue configuration of each logical core */
592 for (portid = 0; portid < nb_ports; portid++) {
593 /* skip ports that are not enabled */
594 if ((lsi_enabled_port_mask & (1 << portid)) == 0)
597 /* get the lcore_id for this port */
598 while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
599 lcore_queue_conf[rx_lcore_id].n_rx_port ==
600 lsi_rx_queue_per_lcore) {
603 if (rx_lcore_id >= RTE_MAX_LCORE)
604 rte_exit(EXIT_FAILURE, "Not enough cores\n");
606 if (qconf != &lcore_queue_conf[rx_lcore_id])
607 /* Assigned a new logical core in the loop above. */
608 qconf = &lcore_queue_conf[rx_lcore_id];
610 qconf->rx_port_list[qconf->n_rx_port] = portid;
612 printf("Lcore %u: RX port %u\n",rx_lcore_id, (unsigned) portid);
615 /* Initialise each port */
616 for (portid = 0; portid < nb_ports; portid++) {
617 struct rte_eth_rxconf rxq_conf;
618 struct rte_eth_txconf txq_conf;
619 struct rte_eth_conf local_port_conf = port_conf;
620 struct rte_eth_dev_info dev_info;
622 /* skip ports that are not enabled */
623 if ((lsi_enabled_port_mask & (1 << portid)) == 0) {
624 printf("Skipping disabled port %u\n", (unsigned) portid);
628 printf("Initializing port %u... ", (unsigned) portid);
631 ret = rte_eth_dev_info_get(portid, &dev_info);
633 rte_exit(EXIT_FAILURE,
634 "Error during getting device (port %u) info: %s\n",
635 portid, strerror(-ret));
637 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
638 local_port_conf.txmode.offloads |=
639 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
640 /* Configure RX and TX queues. 8< */
641 ret = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);
643 rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n",
644 ret, (unsigned) portid);
645 /* >8 End of configure RX and TX queues. */
647 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
650 rte_exit(EXIT_FAILURE,
651 "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%u\n",
652 ret, (unsigned) portid);
654 /* register lsi interrupt callback, need to be after
655 * rte_eth_dev_configure(). if (intr_conf.lsc == 0), no
656 * lsc interrupt will be present, and below callback to
657 * be registered will never be called.
660 /* RTE callback register. 8< */
661 rte_eth_dev_callback_register(portid,
662 RTE_ETH_EVENT_INTR_LSC, lsi_event_callback, NULL);
663 /* >8 End of registering lsi interrupt callback. */
665 ret = rte_eth_macaddr_get(portid,
666 &lsi_ports_eth_addr[portid]);
668 rte_exit(EXIT_FAILURE,
669 "rte_eth_macaddr_get: err=%d, port=%u\n",
670 ret, (unsigned int)portid);
672 /* init one RX queue */
674 rxq_conf = dev_info.default_rxconf;
675 rxq_conf.offloads = local_port_conf.rxmode.offloads;
676 /* RX queue initialization. 8< */
677 ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
678 rte_eth_dev_socket_id(portid),
682 rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d, port=%u\n",
683 ret, (unsigned) portid);
684 /* >8 End of RX queue initialization. */
686 /* init one TX queue logical core on each port. 8< */
688 txq_conf = dev_info.default_txconf;
689 txq_conf.offloads = local_port_conf.txmode.offloads;
690 ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
691 rte_eth_dev_socket_id(portid),
694 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: err=%d,port=%u\n",
695 ret, (unsigned) portid);
696 /* >8 End of init one TX queue. */
698 /* Initialize TX buffers */
699 tx_buffer[portid] = rte_zmalloc_socket("tx_buffer",
700 RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,
701 rte_eth_dev_socket_id(portid));
702 if (tx_buffer[portid] == NULL)
703 rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n",
706 rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST);
708 ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[portid],
709 rte_eth_tx_buffer_count_callback,
710 &port_statistics[portid].dropped);
712 rte_exit(EXIT_FAILURE, "Cannot set error callback for "
713 "tx buffer on port %u\n", (unsigned) portid);
716 ret = rte_eth_dev_start(portid);
718 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, port=%u\n",
719 ret, (unsigned) portid);
722 ret = rte_eth_promiscuous_enable(portid);
724 rte_exit(EXIT_FAILURE,
725 "rte_eth_promiscuous_enable: err=%s, port=%u\n",
726 rte_strerror(-ret), portid);
728 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n",
730 RTE_ETHER_ADDR_BYTES(&lsi_ports_eth_addr[portid]));
732 /* initialize port stats */
733 memset(&port_statistics, 0, sizeof(port_statistics));
736 check_all_ports_link_status(nb_ports, lsi_enabled_port_mask);
738 /* launch per-lcore init on every lcore */
739 rte_eal_mp_remote_launch(lsi_launch_one_lcore, NULL, CALL_MAIN);
740 RTE_LCORE_FOREACH_WORKER(lcore_id) {
741 if (rte_eal_wait_lcore(lcore_id) < 0)
745 /* clean up the EAL */