1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
12 #ifndef RTE_EXEC_ENV_WINDOWS
15 #include <sys/types.h>
19 #include <sys/queue.h>
26 #include <rte_common.h>
27 #include <rte_errno.h>
28 #include <rte_byteorder.h>
30 #include <rte_debug.h>
31 #include <rte_cycles.h>
32 #include <rte_memory.h>
33 #include <rte_memcpy.h>
34 #include <rte_launch.h>
36 #include <rte_alarm.h>
37 #include <rte_per_lcore.h>
38 #include <rte_lcore.h>
39 #include <rte_branch_prediction.h>
40 #include <rte_mempool.h>
41 #include <rte_malloc.h>
43 #include <rte_mbuf_pool_ops.h>
44 #include <rte_interrupts.h>
46 #include <rte_ether.h>
47 #include <rte_ethdev.h>
49 #include <rte_string_fns.h>
51 #include <rte_pmd_ixgbe.h>
54 #include <rte_pdump.h>
57 #include <rte_metrics.h>
58 #ifdef RTE_LIB_BITRATESTATS
59 #include <rte_bitrate.h>
61 #ifdef RTE_LIB_LATENCYSTATS
62 #include <rte_latencystats.h>
64 #ifdef RTE_EXEC_ENV_WINDOWS
71 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
72 #define HUGE_FLAG (0x40000)
74 #define HUGE_FLAG MAP_HUGETLB
77 #ifndef MAP_HUGE_SHIFT
78 /* older kernels (or FreeBSD) will not have this define */
79 #define HUGE_SHIFT (26)
81 #define HUGE_SHIFT MAP_HUGE_SHIFT
84 #define EXTMEM_HEAP_NAME "extmem"
85 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
87 uint16_t verbose_level = 0; /**< Silent by default. */
88 int testpmd_logtype; /**< Log type for testpmd logs */
90 /* use main core for command line ? */
91 uint8_t interactive = 0;
92 uint8_t auto_start = 0;
94 char cmdline_filename[PATH_MAX] = {0};
97 * NUMA support configuration.
98 * When set, the NUMA support attempts to dispatch the allocation of the
99 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
100 * probed ports among the CPU sockets 0 and 1.
101 * Otherwise, all memory is allocated from CPU socket 0.
103 uint8_t numa_support = 1; /**< numa enabled by default */
106 * In UMA mode,all memory is allocated from socket 0 if --socket-num is
109 uint8_t socket_num = UMA_NO_CONFIG;
112 * Select mempool allocation type:
113 * - native: use regular DPDK memory
114 * - anon: use regular DPDK memory to create mempool, but populate using
115 * anonymous memory (may not be IOVA-contiguous)
116 * - xmem: use externally allocated hugepage memory
118 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
121 * Store specified sockets on which memory pool to be used by ports
124 uint8_t port_numa[RTE_MAX_ETHPORTS];
127 * Store specified sockets on which RX ring to be used by ports
130 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
133 * Store specified sockets on which TX ring to be used by ports
136 uint8_t txring_numa[RTE_MAX_ETHPORTS];
139 * Record the Ethernet address of peer target ports to which packets are
141 * Must be instantiated with the ethernet addresses of peer traffic generator
144 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
145 portid_t nb_peer_eth_addrs = 0;
148 * Probed Target Environment.
150 struct rte_port *ports; /**< For all probed ethernet ports. */
151 portid_t nb_ports; /**< Number of probed ethernet ports. */
152 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
153 lcoreid_t nb_lcores; /**< Number of probed logical cores. */
155 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
158 * Test Forwarding Configuration.
159 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
160 * nb_fwd_ports <= nb_cfg_ports <= nb_ports
162 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
163 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
164 portid_t nb_cfg_ports; /**< Number of configured ports. */
165 portid_t nb_fwd_ports; /**< Number of forwarding ports. */
167 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
168 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */
170 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
171 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */
174 * Forwarding engines.
176 struct fwd_engine * fwd_engines[] = {
186 &five_tuple_swap_fwd_engine,
187 #ifdef RTE_LIBRTE_IEEE1588
188 &ieee1588_fwd_engine,
194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
195 uint16_t mempool_flags;
197 struct fwd_config cur_fwd_config;
198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
199 uint32_t retry_enabled;
200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
205 DEFAULT_MBUF_DATA_SIZE
206 }; /**< Mbuf data space size. */
207 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if
208 * specified on command-line. */
209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
211 /** Extended statistics to show. */
212 struct rte_eth_xstat_name *xstats_display;
214 unsigned int xstats_display_num; /**< Size of extended statistics to show */
217 * In container, it cannot terminate the process which running with 'stats-period'
218 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
223 * Max Rx frame size, set by '--max-pkt-len' parameter.
225 uint32_t max_rx_pkt_len;
228 * Configuration of packet segments used to scatter received packets
229 * if some of split features is configured.
231 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
232 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */
233 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
234 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */
237 * Configuration of packet segments used by the "txonly" processing engine.
239 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
240 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
241 TXONLY_DEF_PACKET_LEN,
243 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
245 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
246 /**< Split policy for packets to TX. */
248 uint8_t txonly_multi_flow;
249 /**< Whether multiple flows are generated in TXONLY mode. */
251 uint32_t tx_pkt_times_inter;
252 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
254 uint32_t tx_pkt_times_intra;
255 /**< Timings for send scheduling in TXONLY mode, time between packets. */
257 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
258 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
259 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
260 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
262 /* current configuration is in DCB or not,0 means it is not in DCB mode */
263 uint8_t dcb_config = 0;
266 * Configurable number of RX/TX queues.
268 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
269 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
270 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
273 * Configurable number of RX/TX ring descriptors.
274 * Defaults are supplied by drivers via ethdev.
276 #define RTE_TEST_RX_DESC_DEFAULT 0
277 #define RTE_TEST_TX_DESC_DEFAULT 0
278 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
279 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
281 #define RTE_PMD_PARAM_UNSET -1
283 * Configurable values of RX and TX ring threshold registers.
286 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
287 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
288 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
290 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
291 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
292 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
295 * Configurable value of RX free threshold.
297 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
300 * Configurable value of RX drop enable.
302 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
305 * Configurable value of TX free threshold.
307 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
310 * Configurable value of TX RS bit threshold.
312 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
315 * Configurable value of buffered packets before sending.
317 uint16_t noisy_tx_sw_bufsz;
320 * Configurable value of packet buffer timeout.
322 uint16_t noisy_tx_sw_buf_flush_time;
325 * Configurable value for size of VNF internal memory area
326 * used for simulating noisy neighbour behaviour
328 uint64_t noisy_lkup_mem_sz;
331 * Configurable value of number of random writes done in
332 * VNF simulation memory area.
334 uint64_t noisy_lkup_num_writes;
337 * Configurable value of number of random reads done in
338 * VNF simulation memory area.
340 uint64_t noisy_lkup_num_reads;
343 * Configurable value of number of random reads/writes done in
344 * VNF simulation memory area.
346 uint64_t noisy_lkup_num_reads_writes;
349 * Receive Side Scaling (RSS) configuration.
351 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */
354 * Port topology configuration
356 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
359 * Avoids to flush all the RX streams before starts forwarding.
361 uint8_t no_flush_rx = 0; /* flush by default */
364 * Flow API isolated mode.
366 uint8_t flow_isolate_all;
369 * Avoids to check link status when starting/stopping a port.
371 uint8_t no_link_check = 0; /* check by default */
374 * Don't automatically start all ports in interactive mode.
376 uint8_t no_device_start = 0;
379 * Enable link status change notification
381 uint8_t lsc_interrupt = 1; /* enabled by default */
384 * Enable device removal notification.
386 uint8_t rmv_interrupt = 1; /* enabled by default */
388 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
390 /* After attach, port setup is called on event or by iterator */
391 bool setup_on_probe_event = true;
393 /* Clear ptypes on port initialization. */
394 uint8_t clear_ptypes = true;
396 /* Hairpin ports configuration mode. */
397 uint16_t hairpin_mode;
399 /* Pretty printing of ethdev events */
400 static const char * const eth_event_desc[] = {
401 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
402 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
403 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
404 [RTE_ETH_EVENT_INTR_RESET] = "reset",
405 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
406 [RTE_ETH_EVENT_IPSEC] = "IPsec",
407 [RTE_ETH_EVENT_MACSEC] = "MACsec",
408 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
409 [RTE_ETH_EVENT_NEW] = "device probed",
410 [RTE_ETH_EVENT_DESTROY] = "device released",
411 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
412 [RTE_ETH_EVENT_MAX] = NULL,
416 * Display or mask ether events
417 * Default to all events except VF_MBOX
419 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
420 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
421 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
422 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
423 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
424 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
425 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
426 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
428 * Decide if all memory are locked for performance.
433 * NIC bypass mode configuration options.
436 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
437 /* The NIC bypass watchdog timeout. */
438 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
442 #ifdef RTE_LIB_LATENCYSTATS
445 * Set when latency stats is enabled in the commandline
447 uint8_t latencystats_enabled;
450 * Lcore ID to serive latency statistics.
452 lcoreid_t latencystats_lcore_id = -1;
457 * Ethernet device configuration.
459 struct rte_eth_rxmode rx_mode;
461 struct rte_eth_txmode tx_mode = {
462 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
465 struct rte_eth_fdir_conf fdir_conf = {
466 .mode = RTE_FDIR_MODE_NONE,
467 .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
468 .status = RTE_FDIR_REPORT_STATUS,
470 .vlan_tci_mask = 0xFFEF,
472 .src_ip = 0xFFFFFFFF,
473 .dst_ip = 0xFFFFFFFF,
476 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
477 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
479 .src_port_mask = 0xFFFF,
480 .dst_port_mask = 0xFFFF,
481 .mac_addr_byte_mask = 0xFF,
482 .tunnel_type_mask = 1,
483 .tunnel_id_mask = 0xFFFFFFFF,
488 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
491 * Display zero values by default for xstats
493 uint8_t xstats_hide_zero;
496 * Measure of CPU cycles disabled by default
498 uint8_t record_core_cycles;
501 * Display of RX and TX bursts disabled by default
503 uint8_t record_burst_stats;
506 * Number of ports per shared Rx queue group, 0 disable.
510 unsigned int num_sockets = 0;
511 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
513 #ifdef RTE_LIB_BITRATESTATS
514 /* Bitrate statistics */
515 struct rte_stats_bitrates *bitrate_data;
516 lcoreid_t bitrate_lcore_id;
517 uint8_t bitrate_enabled;
521 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
522 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
526 * hexadecimal bitmask of RX mq mode can be enabled.
528 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
531 * Used to set forced link speed
533 uint32_t eth_link_speed;
536 * ID of the current process in multi-process, used to
537 * configure the queues to be polled.
542 * Number of processes in multi-process, used to
543 * configure the queues to be polled.
545 unsigned int num_procs = 1;
548 eth_rx_metadata_negotiate_mp(uint16_t port_id)
550 uint64_t rx_meta_features = 0;
553 if (!is_proc_primary())
556 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
557 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
558 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
560 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
562 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
563 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
567 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
568 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
572 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
573 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
576 } else if (ret != -ENOTSUP) {
577 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
578 port_id, rte_strerror(-ret));
583 flow_pick_transfer_proxy_mp(uint16_t port_id)
585 struct rte_port *port = &ports[port_id];
588 port->flow_transfer_proxy = port_id;
590 if (!is_proc_primary())
593 ret = rte_flow_pick_transfer_proxy(port_id, &port->flow_transfer_proxy,
596 fprintf(stderr, "Error picking flow transfer proxy for port %u: %s - ignore\n",
597 port_id, rte_strerror(-ret));
602 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
603 const struct rte_eth_conf *dev_conf)
605 if (is_proc_primary())
606 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
612 eth_dev_start_mp(uint16_t port_id)
614 if (is_proc_primary())
615 return rte_eth_dev_start(port_id);
621 eth_dev_stop_mp(uint16_t port_id)
623 if (is_proc_primary())
624 return rte_eth_dev_stop(port_id);
630 mempool_free_mp(struct rte_mempool *mp)
632 if (is_proc_primary())
633 rte_mempool_free(mp);
637 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
639 if (is_proc_primary())
640 return rte_eth_dev_set_mtu(port_id, mtu);
645 /* Forward function declarations */
646 static void setup_attached_port(portid_t pi);
647 static void check_all_ports_link_status(uint32_t port_mask);
648 static int eth_event_callback(portid_t port_id,
649 enum rte_eth_event_type type,
650 void *param, void *ret_param);
651 static void dev_event_callback(const char *device_name,
652 enum rte_dev_event_type type,
654 static void fill_xstats_display_info(void);
657 * Check if all the ports are started.
658 * If yes, return positive value. If not, return zero.
660 static int all_ports_started(void);
663 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
664 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
667 /* Holds the registered mbuf dynamic flags names. */
668 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
672 * Helper function to check if socket is already discovered.
673 * If yes, return positive value. If not, return zero.
676 new_socket_id(unsigned int socket_id)
680 for (i = 0; i < num_sockets; i++) {
681 if (socket_ids[i] == socket_id)
688 * Setup default configuration.
691 set_default_fwd_lcores_config(void)
695 unsigned int sock_num;
698 for (i = 0; i < RTE_MAX_LCORE; i++) {
699 if (!rte_lcore_is_enabled(i))
701 sock_num = rte_lcore_to_socket_id(i);
702 if (new_socket_id(sock_num)) {
703 if (num_sockets >= RTE_MAX_NUMA_NODES) {
704 rte_exit(EXIT_FAILURE,
705 "Total sockets greater than %u\n",
708 socket_ids[num_sockets++] = sock_num;
710 if (i == rte_get_main_lcore())
712 fwd_lcores_cpuids[nb_lc++] = i;
714 nb_lcores = (lcoreid_t) nb_lc;
715 nb_cfg_lcores = nb_lcores;
720 set_def_peer_eth_addrs(void)
724 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
725 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
726 peer_eth_addrs[i].addr_bytes[5] = i;
731 set_default_fwd_ports_config(void)
736 RTE_ETH_FOREACH_DEV(pt_id) {
737 fwd_ports_ids[i++] = pt_id;
739 /* Update sockets info according to the attached device */
740 int socket_id = rte_eth_dev_socket_id(pt_id);
741 if (socket_id >= 0 && new_socket_id(socket_id)) {
742 if (num_sockets >= RTE_MAX_NUMA_NODES) {
743 rte_exit(EXIT_FAILURE,
744 "Total sockets greater than %u\n",
747 socket_ids[num_sockets++] = socket_id;
751 nb_cfg_ports = nb_ports;
752 nb_fwd_ports = nb_ports;
756 set_def_fwd_config(void)
758 set_default_fwd_lcores_config();
759 set_def_peer_eth_addrs();
760 set_default_fwd_ports_config();
763 #ifndef RTE_EXEC_ENV_WINDOWS
764 /* extremely pessimistic estimation of memory required to create a mempool */
766 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
768 unsigned int n_pages, mbuf_per_pg, leftover;
769 uint64_t total_mem, mbuf_mem, obj_sz;
771 /* there is no good way to predict how much space the mempool will
772 * occupy because it will allocate chunks on the fly, and some of those
773 * will come from default DPDK memory while some will come from our
774 * external memory, so just assume 128MB will be enough for everyone.
776 uint64_t hdr_mem = 128 << 20;
778 /* account for possible non-contiguousness */
779 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
781 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
785 mbuf_per_pg = pgsz / obj_sz;
786 leftover = (nb_mbufs % mbuf_per_pg) > 0;
787 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
789 mbuf_mem = n_pages * pgsz;
791 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
793 if (total_mem > SIZE_MAX) {
794 TESTPMD_LOG(ERR, "Memory size too big\n");
797 *out = (size_t)total_mem;
803 pagesz_flags(uint64_t page_sz)
805 /* as per mmap() manpage, all page sizes are log2 of page size
806 * shifted by MAP_HUGE_SHIFT
808 int log2 = rte_log2_u64(page_sz);
810 return (log2 << HUGE_SHIFT);
814 alloc_mem(size_t memsz, size_t pgsz, bool huge)
819 /* allocate anonymous hugepages */
820 flags = MAP_ANONYMOUS | MAP_PRIVATE;
822 flags |= HUGE_FLAG | pagesz_flags(pgsz);
824 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
825 if (addr == MAP_FAILED)
831 struct extmem_param {
835 rte_iova_t *iova_table;
836 unsigned int iova_table_len;
840 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
843 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
844 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */
845 unsigned int cur_page, n_pages, pgsz_idx;
846 size_t mem_sz, cur_pgsz;
847 rte_iova_t *iovas = NULL;
851 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
852 /* skip anything that is too big */
853 if (pgsizes[pgsz_idx] > SIZE_MAX)
856 cur_pgsz = pgsizes[pgsz_idx];
858 /* if we were told not to allocate hugepages, override */
860 cur_pgsz = sysconf(_SC_PAGESIZE);
862 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
864 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
868 /* allocate our memory */
869 addr = alloc_mem(mem_sz, cur_pgsz, huge);
871 /* if we couldn't allocate memory with a specified page size,
872 * that doesn't mean we can't do it with other page sizes, so
878 /* store IOVA addresses for every page in this memory area */
879 n_pages = mem_sz / cur_pgsz;
881 iovas = malloc(sizeof(*iovas) * n_pages);
884 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
887 /* lock memory if it's not huge pages */
891 /* populate IOVA addresses */
892 for (cur_page = 0; cur_page < n_pages; cur_page++) {
897 offset = cur_pgsz * cur_page;
898 cur = RTE_PTR_ADD(addr, offset);
900 /* touch the page before getting its IOVA */
901 *(volatile char *)cur = 0;
903 iova = rte_mem_virt2iova(cur);
905 iovas[cur_page] = iova;
910 /* if we couldn't allocate anything */
916 param->pgsz = cur_pgsz;
917 param->iova_table = iovas;
918 param->iova_table_len = n_pages;
925 munmap(addr, mem_sz);
931 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
933 struct extmem_param param;
936 memset(¶m, 0, sizeof(param));
938 /* check if our heap exists */
939 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
941 /* create our heap */
942 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
944 TESTPMD_LOG(ERR, "Cannot create heap\n");
949 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
951 TESTPMD_LOG(ERR, "Cannot create memory area\n");
955 /* we now have a valid memory area, so add it to heap */
956 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
957 param.addr, param.len, param.iova_table,
958 param.iova_table_len, param.pgsz);
960 /* when using VFIO, memory is automatically mapped for DMA by EAL */
962 /* not needed any more */
963 free(param.iova_table);
966 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
967 munmap(param.addr, param.len);
973 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
979 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
980 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
985 RTE_ETH_FOREACH_DEV(pid) {
986 struct rte_eth_dev_info dev_info;
988 ret = eth_dev_info_get_print_err(pid, &dev_info);
991 "unable to get device info for port %d on addr 0x%p,"
992 "mempool unmapping will not be performed\n",
997 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
1000 "unable to DMA unmap addr 0x%p "
1002 memhdr->addr, dev_info.device->name);
1005 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
1008 "unable to un-register addr 0x%p\n", memhdr->addr);
1013 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1014 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1017 size_t page_size = sysconf(_SC_PAGESIZE);
1020 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1024 "unable to register addr 0x%p\n", memhdr->addr);
1027 RTE_ETH_FOREACH_DEV(pid) {
1028 struct rte_eth_dev_info dev_info;
1030 ret = eth_dev_info_get_print_err(pid, &dev_info);
1033 "unable to get device info for port %d on addr 0x%p,"
1034 "mempool mapping will not be performed\n",
1038 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1041 "unable to DMA map addr 0x%p "
1043 memhdr->addr, dev_info.device->name);
1050 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1051 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1053 struct rte_pktmbuf_extmem *xmem;
1054 unsigned int ext_num, zone_num, elt_num;
1057 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1058 elt_num = EXTBUF_ZONE_SIZE / elt_size;
1059 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1061 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1063 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1064 "external buffer descriptors\n");
1068 for (ext_num = 0; ext_num < zone_num; ext_num++) {
1069 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1070 const struct rte_memzone *mz;
1071 char mz_name[RTE_MEMZONE_NAMESIZE];
1074 ret = snprintf(mz_name, sizeof(mz_name),
1075 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1076 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1077 errno = ENAMETOOLONG;
1081 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
1083 RTE_MEMZONE_IOVA_CONTIG |
1085 RTE_MEMZONE_SIZE_HINT_ONLY,
1089 * The caller exits on external buffer creation
1090 * error, so there is no need to free memzones.
1096 xseg->buf_ptr = mz->addr;
1097 xseg->buf_iova = mz->iova;
1098 xseg->buf_len = EXTBUF_ZONE_SIZE;
1099 xseg->elt_size = elt_size;
1101 if (ext_num == 0 && xmem != NULL) {
1110 * Configuration initialisation done once at init time.
1112 static struct rte_mempool *
1113 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1114 unsigned int socket_id, uint16_t size_idx)
1116 char pool_name[RTE_MEMPOOL_NAMESIZE];
1117 struct rte_mempool *rte_mp = NULL;
1118 #ifndef RTE_EXEC_ENV_WINDOWS
1121 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1123 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1124 if (!is_proc_primary()) {
1125 rte_mp = rte_mempool_lookup(pool_name);
1127 rte_exit(EXIT_FAILURE,
1128 "Get mbuf pool for socket %u failed: %s\n",
1129 socket_id, rte_strerror(rte_errno));
1134 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1135 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1137 switch (mp_alloc_type) {
1138 case MP_ALLOC_NATIVE:
1140 /* wrapper to rte_mempool_create() */
1141 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1142 rte_mbuf_best_mempool_ops());
1143 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1144 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1147 #ifndef RTE_EXEC_ENV_WINDOWS
1150 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1151 mb_size, (unsigned int) mb_mempool_cache,
1152 sizeof(struct rte_pktmbuf_pool_private),
1153 socket_id, mempool_flags);
1157 if (rte_mempool_populate_anon(rte_mp) == 0) {
1158 rte_mempool_free(rte_mp);
1162 rte_pktmbuf_pool_init(rte_mp, NULL);
1163 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1164 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1168 case MP_ALLOC_XMEM_HUGE:
1171 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1173 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1174 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1177 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1178 if (heap_socket < 0)
1179 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1181 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1182 rte_mbuf_best_mempool_ops());
1183 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1184 mb_mempool_cache, 0, mbuf_seg_size,
1191 struct rte_pktmbuf_extmem *ext_mem;
1192 unsigned int ext_num;
1194 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1195 socket_id, pool_name, &ext_mem);
1197 rte_exit(EXIT_FAILURE,
1198 "Can't create pinned data buffers\n");
1200 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1201 rte_mbuf_best_mempool_ops());
1202 rte_mp = rte_pktmbuf_pool_create_extbuf
1203 (pool_name, nb_mbuf, mb_mempool_cache,
1204 0, mbuf_seg_size, socket_id,
1211 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1215 #ifndef RTE_EXEC_ENV_WINDOWS
1218 if (rte_mp == NULL) {
1219 rte_exit(EXIT_FAILURE,
1220 "Creation of mbuf pool for socket %u failed: %s\n",
1221 socket_id, rte_strerror(rte_errno));
1222 } else if (verbose_level > 0) {
1223 rte_mempool_dump(stdout, rte_mp);
1229 * Check given socket id is valid or not with NUMA mode,
1230 * if valid, return 0, else return -1
1233 check_socket_id(const unsigned int socket_id)
1235 static int warning_once = 0;
1237 if (new_socket_id(socket_id)) {
1238 if (!warning_once && numa_support)
1240 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1248 * Get the allowed maximum number of RX queues.
1249 * *pid return the port id which has minimal value of
1250 * max_rx_queues in all ports.
1253 get_allowed_max_nb_rxq(portid_t *pid)
1255 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1256 bool max_rxq_valid = false;
1258 struct rte_eth_dev_info dev_info;
1260 RTE_ETH_FOREACH_DEV(pi) {
1261 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1264 max_rxq_valid = true;
1265 if (dev_info.max_rx_queues < allowed_max_rxq) {
1266 allowed_max_rxq = dev_info.max_rx_queues;
1270 return max_rxq_valid ? allowed_max_rxq : 0;
1274 * Check input rxq is valid or not.
1275 * If input rxq is not greater than any of maximum number
1276 * of RX queues of all ports, it is valid.
1277 * if valid, return 0, else return -1
1280 check_nb_rxq(queueid_t rxq)
1282 queueid_t allowed_max_rxq;
1285 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1286 if (rxq > allowed_max_rxq) {
1288 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1289 rxq, allowed_max_rxq, pid);
1296 * Get the allowed maximum number of TX queues.
1297 * *pid return the port id which has minimal value of
1298 * max_tx_queues in all ports.
1301 get_allowed_max_nb_txq(portid_t *pid)
1303 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1304 bool max_txq_valid = false;
1306 struct rte_eth_dev_info dev_info;
1308 RTE_ETH_FOREACH_DEV(pi) {
1309 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1312 max_txq_valid = true;
1313 if (dev_info.max_tx_queues < allowed_max_txq) {
1314 allowed_max_txq = dev_info.max_tx_queues;
1318 return max_txq_valid ? allowed_max_txq : 0;
1322 * Check input txq is valid or not.
1323 * If input txq is not greater than any of maximum number
1324 * of TX queues of all ports, it is valid.
1325 * if valid, return 0, else return -1
1328 check_nb_txq(queueid_t txq)
1330 queueid_t allowed_max_txq;
1333 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1334 if (txq > allowed_max_txq) {
1336 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1337 txq, allowed_max_txq, pid);
1344 * Get the allowed maximum number of RXDs of every rx queue.
1345 * *pid return the port id which has minimal value of
1346 * max_rxd in all queues of all ports.
1349 get_allowed_max_nb_rxd(portid_t *pid)
1351 uint16_t allowed_max_rxd = UINT16_MAX;
1353 struct rte_eth_dev_info dev_info;
1355 RTE_ETH_FOREACH_DEV(pi) {
1356 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1359 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1360 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1364 return allowed_max_rxd;
1368 * Get the allowed minimal number of RXDs of every rx queue.
1369 * *pid return the port id which has minimal value of
1370 * min_rxd in all queues of all ports.
1373 get_allowed_min_nb_rxd(portid_t *pid)
1375 uint16_t allowed_min_rxd = 0;
1377 struct rte_eth_dev_info dev_info;
1379 RTE_ETH_FOREACH_DEV(pi) {
1380 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1383 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1384 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1389 return allowed_min_rxd;
1393 * Check input rxd is valid or not.
1394 * If input rxd is not greater than any of maximum number
1395 * of RXDs of every Rx queues and is not less than any of
1396 * minimal number of RXDs of every Rx queues, it is valid.
1397 * if valid, return 0, else return -1
1400 check_nb_rxd(queueid_t rxd)
1402 uint16_t allowed_max_rxd;
1403 uint16_t allowed_min_rxd;
1406 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1407 if (rxd > allowed_max_rxd) {
1409 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1410 rxd, allowed_max_rxd, pid);
1414 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1415 if (rxd < allowed_min_rxd) {
1417 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1418 rxd, allowed_min_rxd, pid);
1426 * Get the allowed maximum number of TXDs of every rx queues.
1427 * *pid return the port id which has minimal value of
1428 * max_txd in every tx queue.
1431 get_allowed_max_nb_txd(portid_t *pid)
1433 uint16_t allowed_max_txd = UINT16_MAX;
1435 struct rte_eth_dev_info dev_info;
1437 RTE_ETH_FOREACH_DEV(pi) {
1438 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1441 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1442 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1446 return allowed_max_txd;
1450 * Get the allowed maximum number of TXDs of every tx queues.
1451 * *pid return the port id which has minimal value of
1452 * min_txd in every tx queue.
1455 get_allowed_min_nb_txd(portid_t *pid)
1457 uint16_t allowed_min_txd = 0;
1459 struct rte_eth_dev_info dev_info;
1461 RTE_ETH_FOREACH_DEV(pi) {
1462 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1465 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1466 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1471 return allowed_min_txd;
1475 * Check input txd is valid or not.
1476 * If input txd is not greater than any of maximum number
1477 * of TXDs of every Rx queues, it is valid.
1478 * if valid, return 0, else return -1
1481 check_nb_txd(queueid_t txd)
1483 uint16_t allowed_max_txd;
1484 uint16_t allowed_min_txd;
1487 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1488 if (txd > allowed_max_txd) {
1490 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1491 txd, allowed_max_txd, pid);
1495 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1496 if (txd < allowed_min_txd) {
1498 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1499 txd, allowed_min_txd, pid);
1507 * Get the allowed maximum number of hairpin queues.
1508 * *pid return the port id which has minimal value of
1509 * max_hairpin_queues in all ports.
1512 get_allowed_max_nb_hairpinq(portid_t *pid)
1514 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1516 struct rte_eth_hairpin_cap cap;
1518 RTE_ETH_FOREACH_DEV(pi) {
1519 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1523 if (cap.max_nb_queues < allowed_max_hairpinq) {
1524 allowed_max_hairpinq = cap.max_nb_queues;
1528 return allowed_max_hairpinq;
1532 * Check input hairpin is valid or not.
1533 * If input hairpin is not greater than any of maximum number
1534 * of hairpin queues of all ports, it is valid.
1535 * if valid, return 0, else return -1
1538 check_nb_hairpinq(queueid_t hairpinq)
1540 queueid_t allowed_max_hairpinq;
1543 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1544 if (hairpinq > allowed_max_hairpinq) {
1546 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1547 hairpinq, allowed_max_hairpinq, pid);
1554 get_eth_overhead(struct rte_eth_dev_info *dev_info)
1556 uint32_t eth_overhead;
1558 if (dev_info->max_mtu != UINT16_MAX &&
1559 dev_info->max_rx_pktlen > dev_info->max_mtu)
1560 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1562 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1564 return eth_overhead;
1568 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1570 struct rte_port *port = &ports[pid];
1574 eth_rx_metadata_negotiate_mp(pid);
1575 flow_pick_transfer_proxy_mp(pid);
1577 port->dev_conf.txmode = tx_mode;
1578 port->dev_conf.rxmode = rx_mode;
1580 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1582 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1584 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1585 port->dev_conf.txmode.offloads &=
1586 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1588 /* Apply Rx offloads configuration */
1589 for (i = 0; i < port->dev_info.max_rx_queues; i++)
1590 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1591 /* Apply Tx offloads configuration */
1592 for (i = 0; i < port->dev_info.max_tx_queues; i++)
1593 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1596 port->dev_conf.link_speeds = eth_link_speed;
1599 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1600 get_eth_overhead(&port->dev_info);
1602 /* set flag to initialize port/queue */
1603 port->need_reconfig = 1;
1604 port->need_reconfig_queues = 1;
1605 port->socket_id = socket_id;
1606 port->tx_metadata = 0;
1609 * Check for maximum number of segments per MTU.
1610 * Accordingly update the mbuf data size.
1612 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1613 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1614 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1617 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1618 uint16_t data_size = (mtu + eth_overhead) /
1619 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1620 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1622 if (buffer_size > mbuf_data_size[0]) {
1623 mbuf_data_size[0] = buffer_size;
1624 TESTPMD_LOG(WARNING,
1625 "Configured mbuf size of the first segment %hu\n",
1636 struct rte_mempool *mbp;
1637 unsigned int nb_mbuf_per_pool;
1640 struct rte_gro_param gro_param;
1646 /* Configuration of logical cores. */
1647 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1648 sizeof(struct fwd_lcore *) * nb_lcores,
1649 RTE_CACHE_LINE_SIZE);
1650 if (fwd_lcores == NULL) {
1651 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1652 "failed\n", nb_lcores);
1654 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1655 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1656 sizeof(struct fwd_lcore),
1657 RTE_CACHE_LINE_SIZE);
1658 if (fwd_lcores[lc_id] == NULL) {
1659 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1662 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1665 RTE_ETH_FOREACH_DEV(pid) {
1669 socket_id = port_numa[pid];
1670 if (port_numa[pid] == NUMA_NO_CONFIG) {
1671 socket_id = rte_eth_dev_socket_id(pid);
1674 * if socket_id is invalid,
1675 * set to the first available socket.
1677 if (check_socket_id(socket_id) < 0)
1678 socket_id = socket_ids[0];
1681 socket_id = (socket_num == UMA_NO_CONFIG) ?
1684 /* Apply default TxRx configuration for all ports */
1685 init_config_port_offloads(pid, socket_id);
1688 * Create pools of mbuf.
1689 * If NUMA support is disabled, create a single pool of mbuf in
1690 * socket 0 memory by default.
1691 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1693 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1694 * nb_txd can be configured at run time.
1696 if (param_total_num_mbufs)
1697 nb_mbuf_per_pool = param_total_num_mbufs;
1699 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1700 (nb_lcores * mb_mempool_cache) +
1701 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1702 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1708 for (i = 0; i < num_sockets; i++)
1709 for (j = 0; j < mbuf_data_size_n; j++)
1710 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1711 mbuf_pool_create(mbuf_data_size[j],
1717 for (i = 0; i < mbuf_data_size_n; i++)
1718 mempools[i] = mbuf_pool_create
1721 socket_num == UMA_NO_CONFIG ?
1728 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1729 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1732 * Records which Mbuf pool to use by each logical core, if needed.
1734 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1735 mbp = mbuf_pool_find(
1736 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1739 mbp = mbuf_pool_find(0, 0);
1740 fwd_lcores[lc_id]->mbp = mbp;
1742 /* initialize GSO context */
1743 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1744 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1745 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1746 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1748 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1755 /* create a gro context for each lcore */
1756 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1757 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1758 gro_param.max_item_per_flow = MAX_PKT_BURST;
1759 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1760 gro_param.socket_id = rte_lcore_to_socket_id(
1761 fwd_lcores_cpuids[lc_id]);
1762 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1763 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1764 rte_exit(EXIT_FAILURE,
1765 "rte_gro_ctx_create() failed\n");
1773 reconfig(portid_t new_port_id, unsigned socket_id)
1775 /* Reconfiguration of Ethernet ports. */
1776 init_config_port_offloads(new_port_id, socket_id);
1782 init_fwd_streams(void)
1785 struct rte_port *port;
1786 streamid_t sm_id, nb_fwd_streams_new;
1789 /* set socket id according to numa or not */
1790 RTE_ETH_FOREACH_DEV(pid) {
1792 if (nb_rxq > port->dev_info.max_rx_queues) {
1794 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1795 nb_rxq, port->dev_info.max_rx_queues);
1798 if (nb_txq > port->dev_info.max_tx_queues) {
1800 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1801 nb_txq, port->dev_info.max_tx_queues);
1805 if (port_numa[pid] != NUMA_NO_CONFIG)
1806 port->socket_id = port_numa[pid];
1808 port->socket_id = rte_eth_dev_socket_id(pid);
1811 * if socket_id is invalid,
1812 * set to the first available socket.
1814 if (check_socket_id(port->socket_id) < 0)
1815 port->socket_id = socket_ids[0];
1819 if (socket_num == UMA_NO_CONFIG)
1820 port->socket_id = 0;
1822 port->socket_id = socket_num;
1826 q = RTE_MAX(nb_rxq, nb_txq);
1829 "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1832 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1833 if (nb_fwd_streams_new == nb_fwd_streams)
1836 if (fwd_streams != NULL) {
1837 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1838 if (fwd_streams[sm_id] == NULL)
1840 rte_free(fwd_streams[sm_id]);
1841 fwd_streams[sm_id] = NULL;
1843 rte_free(fwd_streams);
1848 nb_fwd_streams = nb_fwd_streams_new;
1849 if (nb_fwd_streams) {
1850 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1851 sizeof(struct fwd_stream *) * nb_fwd_streams,
1852 RTE_CACHE_LINE_SIZE);
1853 if (fwd_streams == NULL)
1854 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1855 " (struct fwd_stream *)) failed\n",
1858 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1859 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1860 " struct fwd_stream", sizeof(struct fwd_stream),
1861 RTE_CACHE_LINE_SIZE);
1862 if (fwd_streams[sm_id] == NULL)
1863 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1864 "(struct fwd_stream) failed\n");
1872 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1874 uint64_t total_burst, sburst;
1876 uint64_t burst_stats[4];
1877 uint16_t pktnb_stats[4];
1879 int burst_percent[4], sburstp;
1883 * First compute the total number of packet bursts and the
1884 * two highest numbers of bursts of the same number of packets.
1886 memset(&burst_stats, 0x0, sizeof(burst_stats));
1887 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1889 /* Show stats for 0 burst size always */
1890 total_burst = pbs->pkt_burst_spread[0];
1891 burst_stats[0] = pbs->pkt_burst_spread[0];
1894 /* Find the next 2 burst sizes with highest occurrences. */
1895 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1896 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1901 total_burst += nb_burst;
1903 if (nb_burst > burst_stats[1]) {
1904 burst_stats[2] = burst_stats[1];
1905 pktnb_stats[2] = pktnb_stats[1];
1906 burst_stats[1] = nb_burst;
1907 pktnb_stats[1] = nb_pkt;
1908 } else if (nb_burst > burst_stats[2]) {
1909 burst_stats[2] = nb_burst;
1910 pktnb_stats[2] = nb_pkt;
1913 if (total_burst == 0)
1916 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1917 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1919 printf("%d%% of other]\n", 100 - sburstp);
1923 sburst += burst_stats[i];
1924 if (sburst == total_burst) {
1925 printf("%d%% of %d pkts]\n",
1926 100 - sburstp, (int) pktnb_stats[i]);
1931 (double)burst_stats[i] / total_burst * 100;
1932 printf("%d%% of %d pkts + ",
1933 burst_percent[i], (int) pktnb_stats[i]);
1934 sburstp += burst_percent[i];
1939 fwd_stream_stats_display(streamid_t stream_id)
1941 struct fwd_stream *fs;
1942 static const char *fwd_top_stats_border = "-------";
1944 fs = fwd_streams[stream_id];
1945 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1946 (fs->fwd_dropped == 0))
1948 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1949 "TX Port=%2d/Queue=%2d %s\n",
1950 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1951 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1952 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1953 " TX-dropped: %-14"PRIu64,
1954 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1956 /* if checksum mode */
1957 if (cur_fwd_eng == &csum_fwd_engine) {
1958 printf(" RX- bad IP checksum: %-14"PRIu64
1959 " Rx- bad L4 checksum: %-14"PRIu64
1960 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1961 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1962 fs->rx_bad_outer_l4_csum);
1963 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1964 fs->rx_bad_outer_ip_csum);
1969 if (record_burst_stats) {
1970 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1971 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1976 fwd_stats_display(void)
1978 static const char *fwd_stats_border = "----------------------";
1979 static const char *acc_stats_border = "+++++++++++++++";
1981 struct fwd_stream *rx_stream;
1982 struct fwd_stream *tx_stream;
1983 uint64_t tx_dropped;
1984 uint64_t rx_bad_ip_csum;
1985 uint64_t rx_bad_l4_csum;
1986 uint64_t rx_bad_outer_l4_csum;
1987 uint64_t rx_bad_outer_ip_csum;
1988 } ports_stats[RTE_MAX_ETHPORTS];
1989 uint64_t total_rx_dropped = 0;
1990 uint64_t total_tx_dropped = 0;
1991 uint64_t total_rx_nombuf = 0;
1992 struct rte_eth_stats stats;
1993 uint64_t fwd_cycles = 0;
1994 uint64_t total_recv = 0;
1995 uint64_t total_xmit = 0;
1996 struct rte_port *port;
2001 memset(ports_stats, 0, sizeof(ports_stats));
2003 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2004 struct fwd_stream *fs = fwd_streams[sm_id];
2006 if (cur_fwd_config.nb_fwd_streams >
2007 cur_fwd_config.nb_fwd_ports) {
2008 fwd_stream_stats_display(sm_id);
2010 ports_stats[fs->tx_port].tx_stream = fs;
2011 ports_stats[fs->rx_port].rx_stream = fs;
2014 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
2016 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
2017 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2018 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2019 fs->rx_bad_outer_l4_csum;
2020 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2021 fs->rx_bad_outer_ip_csum;
2023 if (record_core_cycles)
2024 fwd_cycles += fs->core_cycles;
2026 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2027 pt_id = fwd_ports_ids[i];
2028 port = &ports[pt_id];
2030 rte_eth_stats_get(pt_id, &stats);
2031 stats.ipackets -= port->stats.ipackets;
2032 stats.opackets -= port->stats.opackets;
2033 stats.ibytes -= port->stats.ibytes;
2034 stats.obytes -= port->stats.obytes;
2035 stats.imissed -= port->stats.imissed;
2036 stats.oerrors -= port->stats.oerrors;
2037 stats.rx_nombuf -= port->stats.rx_nombuf;
2039 total_recv += stats.ipackets;
2040 total_xmit += stats.opackets;
2041 total_rx_dropped += stats.imissed;
2042 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2043 total_tx_dropped += stats.oerrors;
2044 total_rx_nombuf += stats.rx_nombuf;
2046 printf("\n %s Forward statistics for port %-2d %s\n",
2047 fwd_stats_border, pt_id, fwd_stats_border);
2049 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2050 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2051 stats.ipackets + stats.imissed);
2053 if (cur_fwd_eng == &csum_fwd_engine) {
2054 printf(" Bad-ipcsum: %-14"PRIu64
2055 " Bad-l4csum: %-14"PRIu64
2056 "Bad-outer-l4csum: %-14"PRIu64"\n",
2057 ports_stats[pt_id].rx_bad_ip_csum,
2058 ports_stats[pt_id].rx_bad_l4_csum,
2059 ports_stats[pt_id].rx_bad_outer_l4_csum);
2060 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n",
2061 ports_stats[pt_id].rx_bad_outer_ip_csum);
2063 if (stats.ierrors + stats.rx_nombuf > 0) {
2064 printf(" RX-error: %-"PRIu64"\n", stats.ierrors);
2065 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2068 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2069 "TX-total: %-"PRIu64"\n",
2070 stats.opackets, ports_stats[pt_id].tx_dropped,
2071 stats.opackets + ports_stats[pt_id].tx_dropped);
2073 if (record_burst_stats) {
2074 if (ports_stats[pt_id].rx_stream)
2075 pkt_burst_stats_display("RX",
2076 &ports_stats[pt_id].rx_stream->rx_burst_stats);
2077 if (ports_stats[pt_id].tx_stream)
2078 pkt_burst_stats_display("TX",
2079 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2082 printf(" %s--------------------------------%s\n",
2083 fwd_stats_border, fwd_stats_border);
2086 printf("\n %s Accumulated forward statistics for all ports"
2088 acc_stats_border, acc_stats_border);
2089 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2091 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2093 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2094 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2095 if (total_rx_nombuf > 0)
2096 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2097 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
2099 acc_stats_border, acc_stats_border);
2100 if (record_core_cycles) {
2101 #define CYC_PER_MHZ 1E6
2102 if (total_recv > 0 || total_xmit > 0) {
2103 uint64_t total_pkts = 0;
2104 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2105 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2106 total_pkts = total_xmit;
2108 total_pkts = total_recv;
2110 printf("\n CPU cycles/packet=%.2F (total cycles="
2111 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2113 (double) fwd_cycles / total_pkts,
2114 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2115 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2121 fwd_stats_reset(void)
2127 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2128 pt_id = fwd_ports_ids[i];
2129 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2131 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2132 struct fwd_stream *fs = fwd_streams[sm_id];
2136 fs->fwd_dropped = 0;
2137 fs->rx_bad_ip_csum = 0;
2138 fs->rx_bad_l4_csum = 0;
2139 fs->rx_bad_outer_l4_csum = 0;
2140 fs->rx_bad_outer_ip_csum = 0;
2142 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2143 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2144 fs->core_cycles = 0;
2149 flush_fwd_rx_queues(void)
2151 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2158 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2159 uint64_t timer_period;
2161 if (num_procs > 1) {
2162 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2166 /* convert to number of cycles */
2167 timer_period = rte_get_timer_hz(); /* 1 second timeout */
2169 for (j = 0; j < 2; j++) {
2170 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2171 for (rxq = 0; rxq < nb_rxq; rxq++) {
2172 port_id = fwd_ports_ids[rxp];
2174 * testpmd can stuck in the below do while loop
2175 * if rte_eth_rx_burst() always returns nonzero
2176 * packets. So timer is added to exit this loop
2177 * after 1sec timer expiry.
2179 prev_tsc = rte_rdtsc();
2181 nb_rx = rte_eth_rx_burst(port_id, rxq,
2182 pkts_burst, MAX_PKT_BURST);
2183 for (i = 0; i < nb_rx; i++)
2184 rte_pktmbuf_free(pkts_burst[i]);
2186 cur_tsc = rte_rdtsc();
2187 diff_tsc = cur_tsc - prev_tsc;
2188 timer_tsc += diff_tsc;
2189 } while ((nb_rx > 0) &&
2190 (timer_tsc < timer_period));
2194 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2199 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2201 struct fwd_stream **fsm;
2204 #ifdef RTE_LIB_BITRATESTATS
2205 uint64_t tics_per_1sec;
2206 uint64_t tics_datum;
2207 uint64_t tics_current;
2208 uint16_t i, cnt_ports;
2210 cnt_ports = nb_ports;
2211 tics_datum = rte_rdtsc();
2212 tics_per_1sec = rte_get_timer_hz();
2214 fsm = &fwd_streams[fc->stream_idx];
2215 nb_fs = fc->stream_nb;
2217 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2218 (*pkt_fwd)(fsm[sm_id]);
2219 #ifdef RTE_LIB_BITRATESTATS
2220 if (bitrate_enabled != 0 &&
2221 bitrate_lcore_id == rte_lcore_id()) {
2222 tics_current = rte_rdtsc();
2223 if (tics_current - tics_datum >= tics_per_1sec) {
2224 /* Periodic bitrate calculation */
2225 for (i = 0; i < cnt_ports; i++)
2226 rte_stats_bitrate_calc(bitrate_data,
2228 tics_datum = tics_current;
2232 #ifdef RTE_LIB_LATENCYSTATS
2233 if (latencystats_enabled != 0 &&
2234 latencystats_lcore_id == rte_lcore_id())
2235 rte_latencystats_update();
2238 } while (! fc->stopped);
2242 start_pkt_forward_on_core(void *fwd_arg)
2244 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2245 cur_fwd_config.fwd_eng->packet_fwd);
2250 * Run the TXONLY packet forwarding engine to send a single burst of packets.
2251 * Used to start communication flows in network loopback test configurations.
2254 run_one_txonly_burst_on_core(void *fwd_arg)
2256 struct fwd_lcore *fwd_lc;
2257 struct fwd_lcore tmp_lcore;
2259 fwd_lc = (struct fwd_lcore *) fwd_arg;
2260 tmp_lcore = *fwd_lc;
2261 tmp_lcore.stopped = 1;
2262 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2267 * Launch packet forwarding:
2268 * - Setup per-port forwarding context.
2269 * - launch logical cores with their forwarding configuration.
2272 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2278 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2279 lc_id = fwd_lcores_cpuids[i];
2280 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2281 fwd_lcores[i]->stopped = 0;
2282 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2283 fwd_lcores[i], lc_id);
2286 "launch lcore %u failed - diag=%d\n",
2293 * Launch packet forwarding configuration.
2296 start_packet_forwarding(int with_tx_first)
2298 port_fwd_begin_t port_fwd_begin;
2299 port_fwd_end_t port_fwd_end;
2302 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2303 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2305 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2306 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2308 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2309 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2310 (!nb_rxq || !nb_txq))
2311 rte_exit(EXIT_FAILURE,
2312 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2313 cur_fwd_eng->fwd_mode_name);
2315 if (all_ports_started() == 0) {
2316 fprintf(stderr, "Not all ports were started\n");
2319 if (test_done == 0) {
2320 fprintf(stderr, "Packet forwarding already started\n");
2326 pkt_fwd_config_display(&cur_fwd_config);
2327 if (!pkt_fwd_shared_rxq_check())
2330 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2331 if (port_fwd_begin != NULL) {
2332 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2333 if (port_fwd_begin(fwd_ports_ids[i])) {
2335 "Packet forwarding is not ready\n");
2341 if (with_tx_first) {
2342 port_fwd_begin = tx_only_engine.port_fwd_begin;
2343 if (port_fwd_begin != NULL) {
2344 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2345 if (port_fwd_begin(fwd_ports_ids[i])) {
2347 "Packet forwarding is not ready\n");
2357 flush_fwd_rx_queues();
2359 rxtx_config_display();
2362 if (with_tx_first) {
2363 while (with_tx_first--) {
2364 launch_packet_forwarding(
2365 run_one_txonly_burst_on_core);
2366 rte_eal_mp_wait_lcore();
2368 port_fwd_end = tx_only_engine.port_fwd_end;
2369 if (port_fwd_end != NULL) {
2370 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2371 (*port_fwd_end)(fwd_ports_ids[i]);
2374 launch_packet_forwarding(start_pkt_forward_on_core);
2378 stop_packet_forwarding(void)
2380 port_fwd_end_t port_fwd_end;
2386 fprintf(stderr, "Packet forwarding not started\n");
2389 printf("Telling cores to stop...");
2390 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2391 fwd_lcores[lc_id]->stopped = 1;
2392 printf("\nWaiting for lcores to finish...\n");
2393 rte_eal_mp_wait_lcore();
2394 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2395 if (port_fwd_end != NULL) {
2396 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2397 pt_id = fwd_ports_ids[i];
2398 (*port_fwd_end)(pt_id);
2402 fwd_stats_display();
2404 printf("\nDone.\n");
2409 dev_set_link_up(portid_t pid)
2411 if (rte_eth_dev_set_link_up(pid) < 0)
2412 fprintf(stderr, "\nSet link up fail.\n");
2416 dev_set_link_down(portid_t pid)
2418 if (rte_eth_dev_set_link_down(pid) < 0)
2419 fprintf(stderr, "\nSet link down fail.\n");
2423 all_ports_started(void)
2426 struct rte_port *port;
2428 RTE_ETH_FOREACH_DEV(pi) {
2430 /* Check if there is a port which is not started */
2431 if ((port->port_status != RTE_PORT_STARTED) &&
2432 (port->slave_flag == 0))
2436 /* No port is not started */
2441 port_is_stopped(portid_t port_id)
2443 struct rte_port *port = &ports[port_id];
2445 if ((port->port_status != RTE_PORT_STOPPED) &&
2446 (port->slave_flag == 0))
2452 all_ports_stopped(void)
2456 RTE_ETH_FOREACH_DEV(pi) {
2457 if (!port_is_stopped(pi))
2465 port_is_started(portid_t port_id)
2467 if (port_id_is_invalid(port_id, ENABLED_WARN))
2470 if (ports[port_id].port_status != RTE_PORT_STARTED)
2476 /* Configure the Rx and Tx hairpin queues for the selected port. */
2478 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2481 struct rte_eth_hairpin_conf hairpin_conf = {
2486 struct rte_port *port = &ports[pi];
2487 uint16_t peer_rx_port = pi;
2488 uint16_t peer_tx_port = pi;
2489 uint32_t manual = 1;
2490 uint32_t tx_exp = hairpin_mode & 0x10;
2492 if (!(hairpin_mode & 0xf)) {
2496 } else if (hairpin_mode & 0x1) {
2497 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2498 RTE_ETH_DEV_NO_OWNER);
2499 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2500 peer_tx_port = rte_eth_find_next_owned_by(0,
2501 RTE_ETH_DEV_NO_OWNER);
2502 if (p_pi != RTE_MAX_ETHPORTS) {
2503 peer_rx_port = p_pi;
2507 /* Last port will be the peer RX port of the first. */
2508 RTE_ETH_FOREACH_DEV(next_pi)
2509 peer_rx_port = next_pi;
2512 } else if (hairpin_mode & 0x2) {
2514 peer_rx_port = p_pi;
2516 peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2517 RTE_ETH_DEV_NO_OWNER);
2518 if (peer_rx_port >= RTE_MAX_ETHPORTS)
2521 peer_tx_port = peer_rx_port;
2525 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2526 hairpin_conf.peers[0].port = peer_rx_port;
2527 hairpin_conf.peers[0].queue = i + nb_rxq;
2528 hairpin_conf.manual_bind = !!manual;
2529 hairpin_conf.tx_explicit = !!tx_exp;
2530 diag = rte_eth_tx_hairpin_queue_setup
2531 (pi, qi, nb_txd, &hairpin_conf);
2536 /* Fail to setup rx queue, return */
2537 if (port->port_status == RTE_PORT_HANDLING)
2538 port->port_status = RTE_PORT_STOPPED;
2541 "Port %d can not be set back to stopped\n", pi);
2542 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2544 /* try to reconfigure queues next time */
2545 port->need_reconfig_queues = 1;
2548 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2549 hairpin_conf.peers[0].port = peer_tx_port;
2550 hairpin_conf.peers[0].queue = i + nb_txq;
2551 hairpin_conf.manual_bind = !!manual;
2552 hairpin_conf.tx_explicit = !!tx_exp;
2553 diag = rte_eth_rx_hairpin_queue_setup
2554 (pi, qi, nb_rxd, &hairpin_conf);
2559 /* Fail to setup rx queue, return */
2560 if (port->port_status == RTE_PORT_HANDLING)
2561 port->port_status = RTE_PORT_STOPPED;
2564 "Port %d can not be set back to stopped\n", pi);
2565 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2567 /* try to reconfigure queues next time */
2568 port->need_reconfig_queues = 1;
2574 /* Configure the Rx with optional split. */
2576 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2577 uint16_t nb_rx_desc, unsigned int socket_id,
2578 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2580 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2581 unsigned int i, mp_n;
2584 if (rx_pkt_nb_segs <= 1 ||
2585 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2586 rx_conf->rx_seg = NULL;
2587 rx_conf->rx_nseg = 0;
2588 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2589 nb_rx_desc, socket_id,
2593 for (i = 0; i < rx_pkt_nb_segs; i++) {
2594 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2595 struct rte_mempool *mpx;
2597 * Use last valid pool for the segments with number
2598 * exceeding the pool index.
2600 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2601 mpx = mbuf_pool_find(socket_id, mp_n);
2602 /* Handle zero as mbuf data buffer size. */
2603 rx_seg->length = rx_pkt_seg_lengths[i] ?
2604 rx_pkt_seg_lengths[i] :
2605 mbuf_data_size[mp_n];
2606 rx_seg->offset = i < rx_pkt_nb_offs ?
2607 rx_pkt_seg_offsets[i] : 0;
2608 rx_seg->mp = mpx ? mpx : mp;
2610 rx_conf->rx_nseg = rx_pkt_nb_segs;
2611 rx_conf->rx_seg = rx_useg;
2612 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2613 socket_id, rx_conf, NULL);
2614 rx_conf->rx_seg = NULL;
2615 rx_conf->rx_nseg = 0;
2620 alloc_xstats_display_info(portid_t pi)
2622 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2623 uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2624 uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2626 if (xstats_display_num == 0)
2629 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2630 if (*ids_supp == NULL)
2633 *prev_values = calloc(xstats_display_num,
2634 sizeof(**prev_values));
2635 if (*prev_values == NULL)
2636 goto fail_prev_values;
2638 *curr_values = calloc(xstats_display_num,
2639 sizeof(**curr_values));
2640 if (*curr_values == NULL)
2641 goto fail_curr_values;
2643 ports[pi].xstats_info.allocated = true;
2656 free_xstats_display_info(portid_t pi)
2658 if (!ports[pi].xstats_info.allocated)
2660 free(ports[pi].xstats_info.ids_supp);
2661 free(ports[pi].xstats_info.prev_values);
2662 free(ports[pi].xstats_info.curr_values);
2663 ports[pi].xstats_info.allocated = false;
2666 /** Fill helper structures for specified port to show extended statistics. */
2668 fill_xstats_display_info_for_port(portid_t pi)
2670 unsigned int stat, stat_supp;
2671 const char *xstat_name;
2672 struct rte_port *port;
2676 if (xstats_display_num == 0)
2679 if (pi == (portid_t)RTE_PORT_ALL) {
2680 fill_xstats_display_info();
2685 if (port->port_status != RTE_PORT_STARTED)
2688 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2689 rte_exit(EXIT_FAILURE,
2690 "Failed to allocate xstats display memory\n");
2692 ids_supp = port->xstats_info.ids_supp;
2693 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2694 xstat_name = xstats_display[stat].name;
2695 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2696 ids_supp + stat_supp);
2698 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2699 xstat_name, pi, stat);
2705 port->xstats_info.ids_supp_sz = stat_supp;
2708 /** Fill helper structures for all ports to show extended statistics. */
2710 fill_xstats_display_info(void)
2714 if (xstats_display_num == 0)
2717 RTE_ETH_FOREACH_DEV(pi)
2718 fill_xstats_display_info_for_port(pi);
2722 start_port(portid_t pid)
2724 int diag, need_check_link_status = -1;
2726 portid_t p_pi = RTE_MAX_ETHPORTS;
2727 portid_t pl[RTE_MAX_ETHPORTS];
2728 portid_t peer_pl[RTE_MAX_ETHPORTS];
2729 uint16_t cnt_pi = 0;
2730 uint16_t cfg_pi = 0;
2733 struct rte_port *port;
2734 struct rte_eth_hairpin_cap cap;
2736 if (port_id_is_invalid(pid, ENABLED_WARN))
2739 RTE_ETH_FOREACH_DEV(pi) {
2740 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2743 need_check_link_status = 0;
2745 if (port->port_status == RTE_PORT_STOPPED)
2746 port->port_status = RTE_PORT_HANDLING;
2748 fprintf(stderr, "Port %d is now not stopped\n", pi);
2752 if (port->need_reconfig > 0) {
2753 struct rte_eth_conf dev_conf;
2756 port->need_reconfig = 0;
2758 if (flow_isolate_all) {
2759 int ret = port_flow_isolate(pi, 1);
2762 "Failed to apply isolated mode on port %d\n",
2767 configure_rxtx_dump_callbacks(0);
2768 printf("Configuring Port %d (socket %u)\n", pi,
2770 if (nb_hairpinq > 0 &&
2771 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2773 "Port %d doesn't support hairpin queues\n",
2778 /* configure port */
2779 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2780 nb_txq + nb_hairpinq,
2783 if (port->port_status == RTE_PORT_HANDLING)
2784 port->port_status = RTE_PORT_STOPPED;
2787 "Port %d can not be set back to stopped\n",
2789 fprintf(stderr, "Fail to configure port %d\n",
2791 /* try to reconfigure port next time */
2792 port->need_reconfig = 1;
2795 /* get device configuration*/
2797 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2799 "port %d can not get device configuration\n",
2803 /* Apply Rx offloads configuration */
2804 if (dev_conf.rxmode.offloads !=
2805 port->dev_conf.rxmode.offloads) {
2806 port->dev_conf.rxmode.offloads |=
2807 dev_conf.rxmode.offloads;
2809 k < port->dev_info.max_rx_queues;
2811 port->rx_conf[k].offloads |=
2812 dev_conf.rxmode.offloads;
2814 /* Apply Tx offloads configuration */
2815 if (dev_conf.txmode.offloads !=
2816 port->dev_conf.txmode.offloads) {
2817 port->dev_conf.txmode.offloads |=
2818 dev_conf.txmode.offloads;
2820 k < port->dev_info.max_tx_queues;
2822 port->tx_conf[k].offloads |=
2823 dev_conf.txmode.offloads;
2826 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2827 port->need_reconfig_queues = 0;
2828 /* setup tx queues */
2829 for (qi = 0; qi < nb_txq; qi++) {
2830 if ((numa_support) &&
2831 (txring_numa[pi] != NUMA_NO_CONFIG))
2832 diag = rte_eth_tx_queue_setup(pi, qi,
2833 port->nb_tx_desc[qi],
2835 &(port->tx_conf[qi]));
2837 diag = rte_eth_tx_queue_setup(pi, qi,
2838 port->nb_tx_desc[qi],
2840 &(port->tx_conf[qi]));
2845 /* Fail to setup tx queue, return */
2846 if (port->port_status == RTE_PORT_HANDLING)
2847 port->port_status = RTE_PORT_STOPPED;
2850 "Port %d can not be set back to stopped\n",
2853 "Fail to configure port %d tx queues\n",
2855 /* try to reconfigure queues next time */
2856 port->need_reconfig_queues = 1;
2859 for (qi = 0; qi < nb_rxq; qi++) {
2860 /* setup rx queues */
2861 if ((numa_support) &&
2862 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2863 struct rte_mempool * mp =
2865 (rxring_numa[pi], 0);
2868 "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2873 diag = rx_queue_setup(pi, qi,
2874 port->nb_rx_desc[qi],
2876 &(port->rx_conf[qi]),
2879 struct rte_mempool *mp =
2881 (port->socket_id, 0);
2884 "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2888 diag = rx_queue_setup(pi, qi,
2889 port->nb_rx_desc[qi],
2891 &(port->rx_conf[qi]),
2897 /* Fail to setup rx queue, return */
2898 if (port->port_status == RTE_PORT_HANDLING)
2899 port->port_status = RTE_PORT_STOPPED;
2902 "Port %d can not be set back to stopped\n",
2905 "Fail to configure port %d rx queues\n",
2907 /* try to reconfigure queues next time */
2908 port->need_reconfig_queues = 1;
2911 /* setup hairpin queues */
2912 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2915 configure_rxtx_dump_callbacks(verbose_level);
2917 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2921 "Port %d: Failed to disable Ptype parsing\n",
2929 diag = eth_dev_start_mp(pi);
2931 fprintf(stderr, "Fail to start port %d: %s\n",
2932 pi, rte_strerror(-diag));
2934 /* Fail to setup rx queue, return */
2935 if (port->port_status == RTE_PORT_HANDLING)
2936 port->port_status = RTE_PORT_STOPPED;
2939 "Port %d can not be set back to stopped\n",
2944 if (port->port_status == RTE_PORT_HANDLING)
2945 port->port_status = RTE_PORT_STARTED;
2947 fprintf(stderr, "Port %d can not be set into started\n",
2950 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
2951 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
2952 RTE_ETHER_ADDR_BYTES(&port->eth_addr));
2954 /* at least one port started, need checking link status */
2955 need_check_link_status = 1;
2960 if (need_check_link_status == 1 && !no_link_check)
2961 check_all_ports_link_status(RTE_PORT_ALL);
2962 else if (need_check_link_status == 0)
2963 fprintf(stderr, "Please stop the ports first\n");
2965 if (hairpin_mode & 0xf) {
2969 /* bind all started hairpin ports */
2970 for (i = 0; i < cfg_pi; i++) {
2972 /* bind current Tx to all peer Rx */
2973 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2974 RTE_MAX_ETHPORTS, 1);
2977 for (j = 0; j < peer_pi; j++) {
2978 if (!port_is_started(peer_pl[j]))
2980 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2983 "Error during binding hairpin Tx port %u to %u: %s\n",
2985 rte_strerror(-diag));
2989 /* bind all peer Tx to current Rx */
2990 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2991 RTE_MAX_ETHPORTS, 0);
2994 for (j = 0; j < peer_pi; j++) {
2995 if (!port_is_started(peer_pl[j]))
2997 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
3000 "Error during binding hairpin Tx port %u to %u: %s\n",
3002 rte_strerror(-diag));
3009 fill_xstats_display_info_for_port(pid);
3016 stop_port(portid_t pid)
3019 struct rte_port *port;
3020 int need_check_link_status = 0;
3021 portid_t peer_pl[RTE_MAX_ETHPORTS];
3024 if (port_id_is_invalid(pid, ENABLED_WARN))
3027 printf("Stopping ports...\n");
3029 RTE_ETH_FOREACH_DEV(pi) {
3030 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3033 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3035 "Please remove port %d from forwarding configuration.\n",
3040 if (port_is_bonding_slave(pi)) {
3042 "Please remove port %d from bonded device.\n",
3048 if (port->port_status == RTE_PORT_STARTED)
3049 port->port_status = RTE_PORT_HANDLING;
3053 if (hairpin_mode & 0xf) {
3056 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3057 /* unbind all peer Tx from current Rx */
3058 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3059 RTE_MAX_ETHPORTS, 0);
3062 for (j = 0; j < peer_pi; j++) {
3063 if (!port_is_started(peer_pl[j]))
3065 rte_eth_hairpin_unbind(peer_pl[j], pi);
3069 if (port->flow_list)
3070 port_flow_flush(pi);
3072 if (eth_dev_stop_mp(pi) != 0)
3073 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3076 if (port->port_status == RTE_PORT_HANDLING)
3077 port->port_status = RTE_PORT_STOPPED;
3079 fprintf(stderr, "Port %d can not be set into stopped\n",
3081 need_check_link_status = 1;
3083 if (need_check_link_status && !no_link_check)
3084 check_all_ports_link_status(RTE_PORT_ALL);
3090 remove_invalid_ports_in(portid_t *array, portid_t *total)
3093 portid_t new_total = 0;
3095 for (i = 0; i < *total; i++)
3096 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3097 array[new_total] = array[i];
3104 remove_invalid_ports(void)
3106 remove_invalid_ports_in(ports_ids, &nb_ports);
3107 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3108 nb_cfg_ports = nb_fwd_ports;
3112 close_port(portid_t pid)
3115 struct rte_port *port;
3117 if (port_id_is_invalid(pid, ENABLED_WARN))
3120 printf("Closing ports...\n");
3122 RTE_ETH_FOREACH_DEV(pi) {
3123 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3126 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3128 "Please remove port %d from forwarding configuration.\n",
3133 if (port_is_bonding_slave(pi)) {
3135 "Please remove port %d from bonded device.\n",
3141 if (port->port_status == RTE_PORT_CLOSED) {
3142 fprintf(stderr, "Port %d is already closed\n", pi);
3146 if (is_proc_primary()) {
3147 port_flow_flush(pi);
3148 port_flex_item_flush(pi);
3149 rte_eth_dev_close(pi);
3152 free_xstats_display_info(pi);
3155 remove_invalid_ports();
3160 reset_port(portid_t pid)
3164 struct rte_port *port;
3166 if (port_id_is_invalid(pid, ENABLED_WARN))
3169 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3170 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3172 "Can not reset port(s), please stop port(s) first.\n");
3176 printf("Resetting ports...\n");
3178 RTE_ETH_FOREACH_DEV(pi) {
3179 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3182 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3184 "Please remove port %d from forwarding configuration.\n",
3189 if (port_is_bonding_slave(pi)) {
3191 "Please remove port %d from bonded device.\n",
3196 diag = rte_eth_dev_reset(pi);
3199 port->need_reconfig = 1;
3200 port->need_reconfig_queues = 1;
3202 fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3211 attach_port(char *identifier)
3214 struct rte_dev_iterator iterator;
3216 printf("Attaching a new port...\n");
3218 if (identifier == NULL) {
3219 fprintf(stderr, "Invalid parameters are specified\n");
3223 if (rte_dev_probe(identifier) < 0) {
3224 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3228 /* first attach mode: event */
3229 if (setup_on_probe_event) {
3230 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3231 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3232 if (ports[pi].port_status == RTE_PORT_HANDLING &&
3233 ports[pi].need_setup != 0)
3234 setup_attached_port(pi);
3238 /* second attach mode: iterator */
3239 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3240 /* setup ports matching the devargs used for probing */
3241 if (port_is_forwarding(pi))
3242 continue; /* port was already attached before */
3243 setup_attached_port(pi);
3248 setup_attached_port(portid_t pi)
3250 unsigned int socket_id;
3253 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3254 /* if socket_id is invalid, set to the first available socket. */
3255 if (check_socket_id(socket_id) < 0)
3256 socket_id = socket_ids[0];
3257 reconfig(pi, socket_id);
3258 ret = rte_eth_promiscuous_enable(pi);
3261 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3262 pi, rte_strerror(-ret));
3264 ports_ids[nb_ports++] = pi;
3265 fwd_ports_ids[nb_fwd_ports++] = pi;
3266 nb_cfg_ports = nb_fwd_ports;
3267 ports[pi].need_setup = 0;
3268 ports[pi].port_status = RTE_PORT_STOPPED;
3270 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3275 detach_device(struct rte_device *dev)
3280 fprintf(stderr, "Device already removed\n");
3284 printf("Removing a device...\n");
3286 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3287 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3288 if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3289 fprintf(stderr, "Port %u not stopped\n",
3293 port_flow_flush(sibling);
3297 if (rte_dev_remove(dev) < 0) {
3298 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3301 remove_invalid_ports();
3303 printf("Device is detached\n");
3304 printf("Now total ports is %d\n", nb_ports);
3310 detach_port_device(portid_t port_id)
3313 struct rte_eth_dev_info dev_info;
3315 if (port_id_is_invalid(port_id, ENABLED_WARN))
3318 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3319 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3320 fprintf(stderr, "Port not stopped\n");
3323 fprintf(stderr, "Port was not closed\n");
3326 ret = eth_dev_info_get_print_err(port_id, &dev_info);
3329 "Failed to get device info for port %d, not detaching\n",
3333 detach_device(dev_info.device);
3337 detach_devargs(char *identifier)
3339 struct rte_dev_iterator iterator;
3340 struct rte_devargs da;
3343 printf("Removing a device...\n");
3345 memset(&da, 0, sizeof(da));
3346 if (rte_devargs_parsef(&da, "%s", identifier)) {
3347 fprintf(stderr, "cannot parse identifier\n");
3351 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3352 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3353 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3354 fprintf(stderr, "Port %u not stopped\n",
3356 rte_eth_iterator_cleanup(&iterator);
3357 rte_devargs_reset(&da);
3360 port_flow_flush(port_id);
3364 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3365 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3366 da.name, da.bus->name);
3367 rte_devargs_reset(&da);
3371 remove_invalid_ports();
3373 printf("Device %s is detached\n", identifier);
3374 printf("Now total ports is %d\n", nb_ports);
3376 rte_devargs_reset(&da);
3387 stop_packet_forwarding();
3389 #ifndef RTE_EXEC_ENV_WINDOWS
3390 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3392 if (mp_alloc_type == MP_ALLOC_ANON)
3393 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3398 if (ports != NULL) {
3400 RTE_ETH_FOREACH_DEV(pt_id) {
3401 printf("\nStopping port %d...\n", pt_id);
3405 RTE_ETH_FOREACH_DEV(pt_id) {
3406 printf("\nShutting down port %d...\n", pt_id);
3413 ret = rte_dev_event_monitor_stop();
3416 "fail to stop device event monitor.");
3420 ret = rte_dev_event_callback_unregister(NULL,
3421 dev_event_callback, NULL);
3424 "fail to unregister device event callback.\n");
3428 ret = rte_dev_hotplug_handle_disable();
3431 "fail to disable hotplug handling.\n");
3435 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3437 mempool_free_mp(mempools[i]);
3439 free(xstats_display);
3441 printf("\nBye...\n");
3444 typedef void (*cmd_func_t)(void);
3445 struct pmd_test_command {
3446 const char *cmd_name;
3447 cmd_func_t cmd_func;
3450 /* Check the link status of all ports in up to 9s, and print them finally */
3452 check_all_ports_link_status(uint32_t port_mask)
3454 #define CHECK_INTERVAL 100 /* 100ms */
3455 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3457 uint8_t count, all_ports_up, print_flag = 0;
3458 struct rte_eth_link link;
3460 char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3462 printf("Checking link statuses...\n");
3464 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3466 RTE_ETH_FOREACH_DEV(portid) {
3467 if ((port_mask & (1 << portid)) == 0)
3469 memset(&link, 0, sizeof(link));
3470 ret = rte_eth_link_get_nowait(portid, &link);
3473 if (print_flag == 1)
3475 "Port %u link get failed: %s\n",
3476 portid, rte_strerror(-ret));
3479 /* print link status if flag set */
3480 if (print_flag == 1) {
3481 rte_eth_link_to_str(link_status,
3482 sizeof(link_status), &link);
3483 printf("Port %d %s\n", portid, link_status);
3486 /* clear all_ports_up flag if any link down */
3487 if (link.link_status == RTE_ETH_LINK_DOWN) {
3492 /* after finally printing all link status, get out */
3493 if (print_flag == 1)
3496 if (all_ports_up == 0) {
3498 rte_delay_ms(CHECK_INTERVAL);
3501 /* set the print_flag if all ports up or timeout */
3502 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3512 rmv_port_callback(void *arg)
3514 int need_to_start = 0;
3515 int org_no_link_check = no_link_check;
3516 portid_t port_id = (intptr_t)arg;
3517 struct rte_eth_dev_info dev_info;
3520 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3522 if (!test_done && port_is_forwarding(port_id)) {
3524 stop_packet_forwarding();
3528 no_link_check = org_no_link_check;
3530 ret = eth_dev_info_get_print_err(port_id, &dev_info);
3533 "Failed to get device info for port %d, not detaching\n",
3536 struct rte_device *device = dev_info.device;
3537 close_port(port_id);
3538 detach_device(device); /* might be already removed or have more ports */
3541 start_packet_forwarding(0);
3544 /* This function is used by the interrupt thread */
3546 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3549 RTE_SET_USED(param);
3550 RTE_SET_USED(ret_param);
3552 if (type >= RTE_ETH_EVENT_MAX) {
3554 "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3555 port_id, __func__, type);
3557 } else if (event_print_mask & (UINT32_C(1) << type)) {
3558 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3559 eth_event_desc[type]);
3564 case RTE_ETH_EVENT_NEW:
3565 ports[port_id].need_setup = 1;
3566 ports[port_id].port_status = RTE_PORT_HANDLING;
3568 case RTE_ETH_EVENT_INTR_RMV:
3569 if (port_id_is_invalid(port_id, DISABLED_WARN))
3571 if (rte_eal_alarm_set(100000,
3572 rmv_port_callback, (void *)(intptr_t)port_id))
3574 "Could not set up deferred device removal\n");
3576 case RTE_ETH_EVENT_DESTROY:
3577 ports[port_id].port_status = RTE_PORT_CLOSED;
3578 printf("Port %u is closed\n", port_id);
3587 register_eth_event_callback(void)
3590 enum rte_eth_event_type event;
3592 for (event = RTE_ETH_EVENT_UNKNOWN;
3593 event < RTE_ETH_EVENT_MAX; event++) {
3594 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3599 TESTPMD_LOG(ERR, "Failed to register callback for "
3600 "%s event\n", eth_event_desc[event]);
3608 /* This function is used by the interrupt thread */
3610 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3611 __rte_unused void *arg)
3616 if (type >= RTE_DEV_EVENT_MAX) {
3617 fprintf(stderr, "%s called upon invalid event %d\n",
3623 case RTE_DEV_EVENT_REMOVE:
3624 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3626 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3628 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3633 * Because the user's callback is invoked in eal interrupt
3634 * callback, the interrupt callback need to be finished before
3635 * it can be unregistered when detaching device. So finish
3636 * callback soon and use a deferred removal to detach device
3637 * is need. It is a workaround, once the device detaching be
3638 * moved into the eal in the future, the deferred removal could
3641 if (rte_eal_alarm_set(100000,
3642 rmv_port_callback, (void *)(intptr_t)port_id))
3644 "Could not set up deferred device removal\n");
3646 case RTE_DEV_EVENT_ADD:
3647 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3649 /* TODO: After finish kernel driver binding,
3650 * begin to attach port.
3659 rxtx_port_config(portid_t pid)
3663 struct rte_port *port = &ports[pid];
3665 for (qid = 0; qid < nb_rxq; qid++) {
3666 offloads = port->rx_conf[qid].offloads;
3667 port->rx_conf[qid] = port->dev_info.default_rxconf;
3669 if (rxq_share > 0 &&
3670 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3671 /* Non-zero share group to enable RxQ share. */
3672 port->rx_conf[qid].share_group = pid / rxq_share + 1;
3673 port->rx_conf[qid].share_qid = qid; /* Equal mapping. */
3677 port->rx_conf[qid].offloads = offloads;
3679 /* Check if any Rx parameters have been passed */
3680 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3681 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3683 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3684 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3686 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3687 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3689 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3690 port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3692 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3693 port->rx_conf[qid].rx_drop_en = rx_drop_en;
3695 port->nb_rx_desc[qid] = nb_rxd;
3698 for (qid = 0; qid < nb_txq; qid++) {
3699 offloads = port->tx_conf[qid].offloads;
3700 port->tx_conf[qid] = port->dev_info.default_txconf;
3702 port->tx_conf[qid].offloads = offloads;
3704 /* Check if any Tx parameters have been passed */
3705 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3706 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3708 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3709 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3711 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3712 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3714 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3715 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3717 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3718 port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3720 port->nb_tx_desc[qid] = nb_txd;
3725 * Helper function to set MTU from frame size
3727 * port->dev_info should be set before calling this function.
3729 * return 0 on success, negative on error
3732 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3734 struct rte_port *port = &ports[portid];
3735 uint32_t eth_overhead;
3736 uint16_t mtu, new_mtu;
3738 eth_overhead = get_eth_overhead(&port->dev_info);
3740 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3741 printf("Failed to get MTU for port %u\n", portid);
3745 new_mtu = max_rx_pktlen - eth_overhead;
3750 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3752 "Failed to set MTU to %u for port %u\n",
3757 port->dev_conf.rxmode.mtu = new_mtu;
3763 init_port_config(void)
3766 struct rte_port *port;
3769 RTE_ETH_FOREACH_DEV(pid) {
3771 port->dev_conf.fdir_conf = fdir_conf;
3773 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3778 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3779 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3780 rss_hf & port->dev_info.flow_type_rss_offloads;
3782 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3783 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3786 if (port->dcb_flag == 0) {
3787 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3788 port->dev_conf.rxmode.mq_mode =
3789 (enum rte_eth_rx_mq_mode)
3790 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3792 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3793 port->dev_conf.rxmode.offloads &=
3794 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3797 i < port->dev_info.nb_rx_queues;
3799 port->rx_conf[i].offloads &=
3800 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3804 rxtx_port_config(pid);
3806 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3810 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3811 rte_pmd_ixgbe_bypass_init(pid);
3814 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3815 port->dev_conf.intr_conf.lsc = 1;
3816 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3817 port->dev_conf.intr_conf.rmv = 1;
3821 void set_port_slave_flag(portid_t slave_pid)
3823 struct rte_port *port;
3825 port = &ports[slave_pid];
3826 port->slave_flag = 1;
3829 void clear_port_slave_flag(portid_t slave_pid)
3831 struct rte_port *port;
3833 port = &ports[slave_pid];
3834 port->slave_flag = 0;
3837 uint8_t port_is_bonding_slave(portid_t slave_pid)
3839 struct rte_port *port;
3840 struct rte_eth_dev_info dev_info;
3843 port = &ports[slave_pid];
3844 ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3847 "Failed to get device info for port id %d,"
3848 "cannot determine if the port is a bonded slave",
3852 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3857 const uint16_t vlan_tags[] = {
3858 0, 1, 2, 3, 4, 5, 6, 7,
3859 8, 9, 10, 11, 12, 13, 14, 15,
3860 16, 17, 18, 19, 20, 21, 22, 23,
3861 24, 25, 26, 27, 28, 29, 30, 31
3865 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3866 enum dcb_mode_enable dcb_mode,
3867 enum rte_eth_nb_tcs num_tcs,
3872 struct rte_eth_rss_conf rss_conf;
3875 * Builds up the correct configuration for dcb+vt based on the vlan tags array
3876 * given above, and the number of traffic classes available for use.
3878 if (dcb_mode == DCB_VT_ENABLED) {
3879 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3880 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3881 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3882 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3884 /* VMDQ+DCB RX and TX configurations */
3885 vmdq_rx_conf->enable_default_pool = 0;
3886 vmdq_rx_conf->default_pool = 0;
3887 vmdq_rx_conf->nb_queue_pools =
3888 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3889 vmdq_tx_conf->nb_queue_pools =
3890 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3892 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3893 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3894 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3895 vmdq_rx_conf->pool_map[i].pools =
3896 1 << (i % vmdq_rx_conf->nb_queue_pools);
3898 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3899 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3900 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3903 /* set DCB mode of RX and TX of multiple queues */
3904 eth_conf->rxmode.mq_mode =
3905 (enum rte_eth_rx_mq_mode)
3906 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
3907 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
3909 struct rte_eth_dcb_rx_conf *rx_conf =
3910 ð_conf->rx_adv_conf.dcb_rx_conf;
3911 struct rte_eth_dcb_tx_conf *tx_conf =
3912 ð_conf->tx_adv_conf.dcb_tx_conf;
3914 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3916 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3920 rx_conf->nb_tcs = num_tcs;
3921 tx_conf->nb_tcs = num_tcs;
3923 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3924 rx_conf->dcb_tc[i] = i % num_tcs;
3925 tx_conf->dcb_tc[i] = i % num_tcs;
3928 eth_conf->rxmode.mq_mode =
3929 (enum rte_eth_rx_mq_mode)
3930 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
3931 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3932 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
3936 eth_conf->dcb_capability_en =
3937 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
3939 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
3945 init_port_dcb_config(portid_t pid,
3946 enum dcb_mode_enable dcb_mode,
3947 enum rte_eth_nb_tcs num_tcs,
3950 struct rte_eth_conf port_conf;
3951 struct rte_port *rte_port;
3955 if (num_procs > 1) {
3956 printf("The multi-process feature doesn't support dcb.\n");
3959 rte_port = &ports[pid];
3961 /* retain the original device configuration. */
3962 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
3964 /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3965 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3968 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
3969 /* remove RSS HASH offload for DCB in vt mode */
3970 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
3971 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3972 for (i = 0; i < nb_rxq; i++)
3973 rte_port->rx_conf[i].offloads &=
3974 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3977 /* re-configure the device . */
3978 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3982 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3986 /* If dev_info.vmdq_pool_base is greater than 0,
3987 * the queue id of vmdq pools is started after pf queues.
3989 if (dcb_mode == DCB_VT_ENABLED &&
3990 rte_port->dev_info.vmdq_pool_base > 0) {
3992 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
3997 /* Assume the ports in testpmd have the same dcb capability
3998 * and has the same number of rxq and txq in dcb mode
4000 if (dcb_mode == DCB_VT_ENABLED) {
4001 if (rte_port->dev_info.max_vfs > 0) {
4002 nb_rxq = rte_port->dev_info.nb_rx_queues;
4003 nb_txq = rte_port->dev_info.nb_tx_queues;
4005 nb_rxq = rte_port->dev_info.max_rx_queues;
4006 nb_txq = rte_port->dev_info.max_tx_queues;
4009 /*if vt is disabled, use all pf queues */
4010 if (rte_port->dev_info.vmdq_pool_base == 0) {
4011 nb_rxq = rte_port->dev_info.max_rx_queues;
4012 nb_txq = rte_port->dev_info.max_tx_queues;
4014 nb_rxq = (queueid_t)num_tcs;
4015 nb_txq = (queueid_t)num_tcs;
4019 rx_free_thresh = 64;
4021 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4023 rxtx_port_config(pid);
4025 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4026 for (i = 0; i < RTE_DIM(vlan_tags); i++)
4027 rx_vft_set(pid, vlan_tags[i], 1);
4029 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4033 rte_port->dcb_flag = 1;
4035 /* Enter DCB configuration status */
4046 /* Configuration of Ethernet ports. */
4047 ports = rte_zmalloc("testpmd: ports",
4048 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4049 RTE_CACHE_LINE_SIZE);
4050 if (ports == NULL) {
4051 rte_exit(EXIT_FAILURE,
4052 "rte_zmalloc(%d struct rte_port) failed\n",
4055 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4056 ports[i].xstats_info.allocated = false;
4057 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4058 LIST_INIT(&ports[i].flow_tunnel_list);
4059 /* Initialize ports NUMA structures */
4060 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4061 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4062 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4076 const char clr[] = { 27, '[', '2', 'J', '\0' };
4077 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4079 /* Clear screen and move to top left */
4080 printf("%s%s", clr, top_left);
4082 printf("\nPort statistics ====================================");
4083 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4084 nic_stats_display(fwd_ports_ids[i]);
4090 signal_handler(int signum)
4092 if (signum == SIGINT || signum == SIGTERM) {
4093 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4095 #ifdef RTE_LIB_PDUMP
4096 /* uninitialize packet capture framework */
4099 #ifdef RTE_LIB_LATENCYSTATS
4100 if (latencystats_enabled != 0)
4101 rte_latencystats_uninit();
4104 /* Set flag to indicate the force termination. */
4106 /* exit with the expected status */
4107 #ifndef RTE_EXEC_ENV_WINDOWS
4108 signal(signum, SIG_DFL);
4109 kill(getpid(), signum);
4115 main(int argc, char** argv)
4122 signal(SIGINT, signal_handler);
4123 signal(SIGTERM, signal_handler);
4125 testpmd_logtype = rte_log_register("testpmd");
4126 if (testpmd_logtype < 0)
4127 rte_exit(EXIT_FAILURE, "Cannot register log type");
4128 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4130 diag = rte_eal_init(argc, argv);
4132 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4133 rte_strerror(rte_errno));
4135 ret = register_eth_event_callback();
4137 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4139 #ifdef RTE_LIB_PDUMP
4140 /* initialize packet capture framework */
4145 RTE_ETH_FOREACH_DEV(port_id) {
4146 ports_ids[count] = port_id;
4149 nb_ports = (portid_t) count;
4151 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4153 /* allocate port structures, and init them */
4156 set_def_fwd_config();
4158 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4159 "Check the core mask argument\n");
4161 /* Bitrate/latency stats disabled by default */
4162 #ifdef RTE_LIB_BITRATESTATS
4163 bitrate_enabled = 0;
4165 #ifdef RTE_LIB_LATENCYSTATS
4166 latencystats_enabled = 0;
4169 /* on FreeBSD, mlockall() is disabled by default */
4170 #ifdef RTE_EXEC_ENV_FREEBSD
4179 launch_args_parse(argc, argv);
4181 #ifndef RTE_EXEC_ENV_WINDOWS
4182 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4183 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4188 if (tx_first && interactive)
4189 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4190 "interactive mode.\n");
4192 if (tx_first && lsc_interrupt) {
4194 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4198 if (!nb_rxq && !nb_txq)
4200 "Warning: Either rx or tx queues should be non-zero\n");
4202 if (nb_rxq > 1 && nb_rxq > nb_txq)
4204 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4210 ret = rte_dev_hotplug_handle_enable();
4213 "fail to enable hotplug handling.");
4217 ret = rte_dev_event_monitor_start();
4220 "fail to start device event monitoring.");
4224 ret = rte_dev_event_callback_register(NULL,
4225 dev_event_callback, NULL);
4228 "fail to register device event callback\n");
4233 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4234 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4236 /* set all ports to promiscuous mode by default */
4237 RTE_ETH_FOREACH_DEV(port_id) {
4238 ret = rte_eth_promiscuous_enable(port_id);
4241 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4242 port_id, rte_strerror(-ret));
4245 /* Init metrics library */
4246 rte_metrics_init(rte_socket_id());
4248 #ifdef RTE_LIB_LATENCYSTATS
4249 if (latencystats_enabled != 0) {
4250 int ret = rte_latencystats_init(1, NULL);
4253 "Warning: latencystats init() returned error %d\n",
4255 fprintf(stderr, "Latencystats running on lcore %d\n",
4256 latencystats_lcore_id);
4260 /* Setup bitrate stats */
4261 #ifdef RTE_LIB_BITRATESTATS
4262 if (bitrate_enabled != 0) {
4263 bitrate_data = rte_stats_bitrate_create();
4264 if (bitrate_data == NULL)
4265 rte_exit(EXIT_FAILURE,
4266 "Could not allocate bitrate data.\n");
4267 rte_stats_bitrate_reg(bitrate_data);
4270 #ifdef RTE_LIB_CMDLINE
4271 if (strlen(cmdline_filename) != 0)
4272 cmdline_read_from_file(cmdline_filename);
4274 if (interactive == 1) {
4276 printf("Start automatic packet forwarding\n");
4277 start_packet_forwarding(0);
4289 printf("No commandline core given, start packet forwarding\n");
4290 start_packet_forwarding(tx_first);
4291 if (stats_period != 0) {
4292 uint64_t prev_time = 0, cur_time, diff_time = 0;
4293 uint64_t timer_period;
4295 /* Convert to number of cycles */
4296 timer_period = stats_period * rte_get_timer_hz();
4298 while (f_quit == 0) {
4299 cur_time = rte_get_timer_cycles();
4300 diff_time += cur_time - prev_time;
4302 if (diff_time >= timer_period) {
4304 /* Reset the timer */
4307 /* Sleep to avoid unnecessary checks */
4308 prev_time = cur_time;
4309 rte_delay_us_sleep(US_PER_S);
4313 printf("Press enter to exit\n");
4314 rc = read(0, &c, 1);
4320 ret = rte_eal_cleanup();
4322 rte_exit(EXIT_FAILURE,
4323 "EAL cleanup failed: %s\n", strerror(-ret));
4325 return EXIT_SUCCESS;