4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 #include <sys/types.h>
40 #include <sys/queue.h>
41 #include <netinet/in.h>
49 #include <rte_common.h>
51 #include <rte_malloc.h>
52 #include <rte_memory.h>
53 #include <rte_memcpy.h>
54 #include <rte_memzone.h>
56 #include <rte_launch.h>
57 #include <rte_atomic.h>
58 #include <rte_cycles.h>
59 #include <rte_prefetch.h>
60 #include <rte_lcore.h>
61 #include <rte_per_lcore.h>
62 #include <rte_branch_prediction.h>
63 #include <rte_interrupts.h>
65 #include <rte_random.h>
66 #include <rte_debug.h>
67 #include <rte_ether.h>
68 #include <rte_ethdev.h>
69 #include <rte_mempool.h>
71 #include <rte_timer.h>
72 #include <rte_keepalive.h>
76 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
80 #define MAX_PKT_BURST 32
81 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
84 * Configurable number of RX/TX ring descriptors
86 #define RTE_TEST_RX_DESC_DEFAULT 128
87 #define RTE_TEST_TX_DESC_DEFAULT 512
88 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
89 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
91 /* ethernet addresses of ports */
92 static struct ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS];
94 /* mask of enabled ports */
95 static uint32_t l2fwd_enabled_port_mask;
97 /* list of enabled ports */
98 static uint32_t l2fwd_dst_ports[RTE_MAX_ETHPORTS];
100 static unsigned int l2fwd_rx_queue_per_lcore = 1;
102 #define MAX_RX_QUEUE_PER_LCORE 16
103 #define MAX_TX_QUEUE_PER_PORT 16
104 struct lcore_queue_conf {
106 unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE];
107 } __rte_cache_aligned;
108 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
110 struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
112 static const struct rte_eth_conf port_conf = {
115 .header_split = 0, /**< Header Split disabled */
116 .hw_ip_checksum = 0, /**< IP checksum offload disabled */
117 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
118 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */
119 .hw_strip_crc = 1, /**< CRC stripped by hardware */
122 .mq_mode = ETH_MQ_TX_NONE,
126 struct rte_mempool *l2fwd_pktmbuf_pool = NULL;
128 /* Per-port statistics struct */
129 struct l2fwd_port_statistics {
133 } __rte_cache_aligned;
134 struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS];
136 /* A tsc-based timer responsible for triggering statistics printout */
137 #define TIMER_MILLISECOND 1
138 #define MAX_TIMER_PERIOD 86400 /* 1 day max */
139 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; /* 10 seconds */
140 static int64_t check_period = 5; /* default check cycle is 5ms */
142 /* Keepalive structure */
143 struct rte_keepalive *rte_global_keepalive_info;
145 /* Termination signalling */
146 static int terminate_signal_received;
148 /* Termination signal handler */
149 static void handle_sigterm(__rte_unused int value)
151 terminate_signal_received = 1;
154 /* Print out statistics on packets dropped */
156 print_stats(__attribute__((unused)) struct rte_timer *ptr_timer,
157 __attribute__((unused)) void *ptr_data)
159 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
162 total_packets_dropped = 0;
163 total_packets_tx = 0;
164 total_packets_rx = 0;
166 const char clr[] = { 27, '[', '2', 'J', '\0' };
167 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
169 /* Clear screen and move to top left */
170 printf("%s%s", clr, topLeft);
172 printf("\nPort statistics ====================================");
174 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
175 /* skip disabled ports */
176 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
178 printf("\nStatistics for port %u ------------------------------"
179 "\nPackets sent: %24"PRIu64
180 "\nPackets received: %20"PRIu64
181 "\nPackets dropped: %21"PRIu64,
183 port_statistics[portid].tx,
184 port_statistics[portid].rx,
185 port_statistics[portid].dropped);
187 total_packets_dropped += port_statistics[portid].dropped;
188 total_packets_tx += port_statistics[portid].tx;
189 total_packets_rx += port_statistics[portid].rx;
191 printf("\nAggregate statistics ==============================="
192 "\nTotal packets sent: %18"PRIu64
193 "\nTotal packets received: %14"PRIu64
194 "\nTotal packets dropped: %15"PRIu64,
197 total_packets_dropped);
198 printf("\n====================================================\n");
202 l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid)
204 struct ether_hdr *eth;
208 struct rte_eth_dev_tx_buffer *buffer;
210 dst_port = l2fwd_dst_ports[portid];
211 eth = rte_pktmbuf_mtod(m, struct ether_hdr *);
213 /* 02:00:00:00:00:xx */
214 tmp = ð->d_addr.addr_bytes[0];
215 *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40);
218 ether_addr_copy(&l2fwd_ports_eth_addr[dst_port], ð->s_addr);
220 buffer = tx_buffer[dst_port];
221 sent = rte_eth_tx_buffer(dst_port, 0, buffer, m);
223 port_statistics[dst_port].tx += sent;
226 /* main processing loop */
228 l2fwd_main_loop(void)
230 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
234 uint64_t prev_tsc, diff_tsc, cur_tsc;
235 unsigned i, j, portid, nb_rx;
236 struct lcore_queue_conf *qconf;
237 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
238 / US_PER_S * BURST_TX_DRAIN_US;
239 struct rte_eth_dev_tx_buffer *buffer;
243 lcore_id = rte_lcore_id();
244 qconf = &lcore_queue_conf[lcore_id];
246 if (qconf->n_rx_port == 0) {
247 RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id);
251 RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id);
253 for (i = 0; i < qconf->n_rx_port; i++) {
255 portid = qconf->rx_port_list[i];
256 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id,
260 uint64_t tsc_initial = rte_rdtsc();
261 uint64_t tsc_lifetime = (rand()&0x07) * rte_get_tsc_hz();
263 while (!terminate_signal_received) {
264 /* Keepalive heartbeat */
265 rte_keepalive_mark_alive(rte_global_keepalive_info);
267 cur_tsc = rte_rdtsc();
270 * Die randomly within 7 secs for demo purposes if
273 if (check_period > 0 && cur_tsc - tsc_initial > tsc_lifetime)
277 * TX burst queue drain
279 diff_tsc = cur_tsc - prev_tsc;
280 if (unlikely(diff_tsc > drain_tsc)) {
282 for (i = 0; i < qconf->n_rx_port; i++) {
284 portid = l2fwd_dst_ports[qconf->rx_port_list[i]];
285 buffer = tx_buffer[portid];
287 sent = rte_eth_tx_buffer_flush(portid, 0, buffer);
289 port_statistics[portid].tx += sent;
297 * Read packet from RX queues
299 for (i = 0; i < qconf->n_rx_port; i++) {
301 portid = qconf->rx_port_list[i];
302 nb_rx = rte_eth_rx_burst(portid, 0,
303 pkts_burst, MAX_PKT_BURST);
305 port_statistics[portid].rx += nb_rx;
307 for (j = 0; j < nb_rx; j++) {
309 rte_prefetch0(rte_pktmbuf_mtod(m, void *));
310 l2fwd_simple_forward(m, portid);
317 l2fwd_launch_one_lcore(__attribute__((unused)) void *dummy)
325 l2fwd_usage(const char *prgname)
327 printf("%s [EAL options] -- -p PORTMASK [-q NQ]\n"
328 " -p PORTMASK: hexadecimal bitmask of ports to configure\n"
329 " -q NQ: number of queue (=ports) per lcore (default is 1)\n"
330 " -K PERIOD: Keepalive check period (5 default; 86400 max)\n"
331 " -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n",
336 l2fwd_parse_portmask(const char *portmask)
341 /* parse hexadecimal string */
342 pm = strtoul(portmask, &end, 16);
343 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
353 l2fwd_parse_nqueue(const char *q_arg)
358 /* parse hexadecimal string */
359 n = strtoul(q_arg, &end, 10);
360 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
364 if (n >= MAX_RX_QUEUE_PER_LCORE)
371 l2fwd_parse_timer_period(const char *q_arg)
376 /* parse number string */
377 n = strtol(q_arg, &end, 10);
378 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
380 if (n >= MAX_TIMER_PERIOD)
387 l2fwd_parse_check_period(const char *q_arg)
392 /* parse number string */
393 n = strtol(q_arg, &end, 10);
394 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
396 if (n >= MAX_TIMER_PERIOD)
402 /* Parse the argument given in the command line of the application */
404 l2fwd_parse_args(int argc, char **argv)
409 char *prgname = argv[0];
410 static struct option lgopts[] = {
416 while ((opt = getopt_long(argc, argvopt, "p:q:T:K:",
417 lgopts, &option_index)) != EOF) {
422 l2fwd_enabled_port_mask = l2fwd_parse_portmask(optarg);
423 if (l2fwd_enabled_port_mask == 0) {
424 printf("invalid portmask\n");
425 l2fwd_usage(prgname);
432 l2fwd_rx_queue_per_lcore = l2fwd_parse_nqueue(optarg);
433 if (l2fwd_rx_queue_per_lcore == 0) {
434 printf("invalid queue number\n");
435 l2fwd_usage(prgname);
442 timer_period = l2fwd_parse_timer_period(optarg)
443 * (int64_t)(1000 * TIMER_MILLISECOND);
444 if (timer_period < 0) {
445 printf("invalid timer period\n");
446 l2fwd_usage(prgname);
453 check_period = l2fwd_parse_check_period(optarg);
454 if (check_period < 0) {
455 printf("invalid check period\n");
456 l2fwd_usage(prgname);
463 l2fwd_usage(prgname);
467 l2fwd_usage(prgname);
473 argv[optind-1] = prgname;
476 optind = 1; /* reset getopt lib */
480 /* Check the link status of all ports in up to 9s, and print them finally */
482 check_all_ports_link_status(uint16_t port_num, uint32_t port_mask)
484 #define CHECK_INTERVAL 100 /* 100ms */
485 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
487 uint8_t count, all_ports_up, print_flag = 0;
488 struct rte_eth_link link;
490 printf("\nChecking link status");
492 for (count = 0; count <= MAX_CHECK_TIME; count++) {
494 for (portid = 0; portid < port_num; portid++) {
495 if ((port_mask & (1 << portid)) == 0)
497 memset(&link, 0, sizeof(link));
498 rte_eth_link_get_nowait(portid, &link);
499 /* print link status if flag set */
500 if (print_flag == 1) {
501 if (link.link_status)
503 "Port%d Link Up. Speed %u Mbps - %s\n",
504 portid, link.link_speed,
505 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
506 ("full-duplex") : ("half-duplex\n"));
508 printf("Port %d Link Down\n", portid);
511 /* clear all_ports_up flag if any link down */
512 if (link.link_status == ETH_LINK_DOWN) {
517 /* after finally printing all link status, get out */
521 if (all_ports_up == 0) {
524 rte_delay_ms(CHECK_INTERVAL);
527 /* set the print_flag if all ports up or timeout */
528 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
536 dead_core(__rte_unused void *ptr_data, const int id_core)
538 if (terminate_signal_received)
540 printf("Dead core %i - restarting..\n", id_core);
541 if (rte_eal_get_lcore_state(id_core) == FINISHED) {
542 rte_eal_wait_lcore(id_core);
543 rte_eal_remote_launch(l2fwd_launch_one_lcore, NULL, id_core);
545 printf("..false positive!\n");
550 relay_core_state(void *ptr_data, const int id_core,
551 const enum rte_keepalive_state core_state, uint64_t last_alive)
553 rte_keepalive_relayed_state((struct rte_keepalive_shm *)ptr_data,
554 id_core, core_state, last_alive);
558 main(int argc, char **argv)
560 struct lcore_queue_conf *qconf;
561 struct rte_eth_dev_info dev_info;
564 uint16_t nb_ports_available;
565 uint16_t portid, last_port;
566 unsigned lcore_id, rx_lcore_id;
567 unsigned nb_ports_in_mask = 0;
568 struct sigaction signal_handler;
569 struct rte_keepalive_shm *ka_shm;
571 memset(&signal_handler, 0, sizeof(signal_handler));
572 terminate_signal_received = 0;
573 signal_handler.sa_handler = &handle_sigterm;
574 if (sigaction(SIGINT, &signal_handler, NULL) == -1 ||
575 sigaction(SIGTERM, &signal_handler, NULL) == -1)
576 rte_exit(EXIT_FAILURE, "SIGNAL\n");
580 ret = rte_eal_init(argc, argv);
582 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
586 l2fwd_enabled_port_mask = 0;
588 /* parse application arguments (after the EAL ones) */
589 ret = l2fwd_parse_args(argc, argv);
591 rte_exit(EXIT_FAILURE, "Invalid L2FWD arguments\n");
593 /* create the mbuf pool */
594 l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 32,
595 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
596 if (l2fwd_pktmbuf_pool == NULL)
597 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n");
599 nb_ports = rte_eth_dev_count();
601 rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");
603 /* reset l2fwd_dst_ports */
604 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++)
605 l2fwd_dst_ports[portid] = 0;
609 * Each logical core is assigned a dedicated TX queue on each port.
611 for (portid = 0; portid < nb_ports; portid++) {
612 /* skip ports that are not enabled */
613 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
616 if (nb_ports_in_mask % 2) {
617 l2fwd_dst_ports[portid] = last_port;
618 l2fwd_dst_ports[last_port] = portid;
624 rte_eth_dev_info_get(portid, &dev_info);
626 if (nb_ports_in_mask % 2) {
627 printf("Notice: odd number of ports in portmask.\n");
628 l2fwd_dst_ports[last_port] = last_port;
634 /* Initialize the port/queue configuration of each logical core */
635 for (portid = 0; portid < nb_ports; portid++) {
636 /* skip ports that are not enabled */
637 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
640 /* get the lcore_id for this port */
641 while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
642 lcore_queue_conf[rx_lcore_id].n_rx_port ==
643 l2fwd_rx_queue_per_lcore) {
645 if (rx_lcore_id >= RTE_MAX_LCORE)
646 rte_exit(EXIT_FAILURE, "Not enough cores\n");
649 if (qconf != &lcore_queue_conf[rx_lcore_id])
650 /* Assigned a new logical core in the loop above. */
651 qconf = &lcore_queue_conf[rx_lcore_id];
653 qconf->rx_port_list[qconf->n_rx_port] = portid;
655 printf("Lcore %u: RX port %u\n",
656 rx_lcore_id, portid);
659 nb_ports_available = nb_ports;
661 /* Initialise each port */
662 for (portid = 0; portid < nb_ports; portid++) {
663 /* skip ports that are not enabled */
664 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) {
665 printf("Skipping disabled port %u\n", portid);
666 nb_ports_available--;
670 printf("Initializing port %u... ", portid);
672 ret = rte_eth_dev_configure(portid, 1, 1, &port_conf);
674 rte_exit(EXIT_FAILURE,
675 "Cannot configure device: err=%d, port=%u\n",
678 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
681 rte_exit(EXIT_FAILURE,
682 "Cannot adjust number of descriptors: err=%d, port=%u\n",
685 rte_eth_macaddr_get(portid, &l2fwd_ports_eth_addr[portid]);
687 /* init one RX queue */
689 ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
690 rte_eth_dev_socket_id(portid),
694 rte_exit(EXIT_FAILURE,
695 "rte_eth_rx_queue_setup:err=%d, port=%u\n",
698 /* init one TX queue on each port */
700 ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
701 rte_eth_dev_socket_id(portid),
704 rte_exit(EXIT_FAILURE,
705 "rte_eth_tx_queue_setup:err=%d, port=%u\n",
708 /* Initialize TX buffers */
709 tx_buffer[portid] = rte_zmalloc_socket("tx_buffer",
710 RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,
711 rte_eth_dev_socket_id(portid));
712 if (tx_buffer[portid] == NULL)
713 rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n",
716 rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST);
718 ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[portid],
719 rte_eth_tx_buffer_count_callback,
720 &port_statistics[portid].dropped);
722 rte_exit(EXIT_FAILURE,
723 "Cannot set error callback for tx buffer on port %u\n",
727 ret = rte_eth_dev_start(portid);
729 rte_exit(EXIT_FAILURE,
730 "rte_eth_dev_start:err=%d, port=%u\n",
733 rte_eth_promiscuous_enable(portid);
735 printf("Port %u, MAC address: "
736 "%02X:%02X:%02X:%02X:%02X:%02X\n\n",
738 l2fwd_ports_eth_addr[portid].addr_bytes[0],
739 l2fwd_ports_eth_addr[portid].addr_bytes[1],
740 l2fwd_ports_eth_addr[portid].addr_bytes[2],
741 l2fwd_ports_eth_addr[portid].addr_bytes[3],
742 l2fwd_ports_eth_addr[portid].addr_bytes[4],
743 l2fwd_ports_eth_addr[portid].addr_bytes[5]);
745 /* initialize port stats */
746 memset(&port_statistics, 0, sizeof(port_statistics));
749 if (!nb_ports_available) {
750 rte_exit(EXIT_FAILURE,
751 "All available ports are disabled. Please set portmask.\n");
754 check_all_ports_link_status(nb_ports, l2fwd_enabled_port_mask);
756 struct rte_timer hb_timer, stats_timer;
758 rte_timer_subsystem_init();
759 rte_timer_init(&stats_timer);
762 if (check_period > 0) {
763 ka_shm = rte_keepalive_shm_create();
765 rte_exit(EXIT_FAILURE,
766 "rte_keepalive_shm_create() failed");
767 rte_global_keepalive_info =
768 rte_keepalive_create(&dead_core, ka_shm);
769 if (rte_global_keepalive_info == NULL)
770 rte_exit(EXIT_FAILURE, "init_keep_alive() failed");
771 rte_keepalive_register_relay_callback(rte_global_keepalive_info,
772 relay_core_state, ka_shm);
773 rte_timer_init(&hb_timer);
774 if (rte_timer_reset(&hb_timer,
775 (check_period * rte_get_timer_hz()) / 1000,
778 (void(*)(struct rte_timer*, void*))
779 &rte_keepalive_dispatch_pings,
780 rte_global_keepalive_info
782 rte_exit(EXIT_FAILURE, "Keepalive setup failure.\n");
784 if (timer_period > 0) {
785 if (rte_timer_reset(&stats_timer,
786 (timer_period * rte_get_timer_hz()) / 1000,
791 rte_exit(EXIT_FAILURE, "Stats setup failure.\n");
793 /* launch per-lcore init on every slave lcore */
794 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
795 struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id];
797 if (qconf->n_rx_port == 0)
799 "lcore %u has nothing to do\n",
803 rte_eal_remote_launch(
804 l2fwd_launch_one_lcore,
808 rte_keepalive_register_core(rte_global_keepalive_info,
812 while (!terminate_signal_received) {
817 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
818 if (rte_eal_wait_lcore(lcore_id) < 0)
823 rte_keepalive_shm_cleanup(ka_shm);