1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
12 #ifndef RTE_EXEC_ENV_WINDOWS
15 #include <sys/types.h>
19 #include <sys/queue.h>
26 #include <rte_common.h>
27 #include <rte_errno.h>
28 #include <rte_byteorder.h>
30 #include <rte_debug.h>
31 #include <rte_cycles.h>
32 #include <rte_memory.h>
33 #include <rte_memcpy.h>
34 #include <rte_launch.h>
36 #include <rte_alarm.h>
37 #include <rte_per_lcore.h>
38 #include <rte_lcore.h>
39 #include <rte_branch_prediction.h>
40 #include <rte_mempool.h>
41 #include <rte_malloc.h>
43 #include <rte_mbuf_pool_ops.h>
44 #include <rte_interrupts.h>
46 #include <rte_ether.h>
47 #include <rte_ethdev.h>
49 #include <rte_string_fns.h>
51 #include <rte_pmd_ixgbe.h>
54 #include <rte_pdump.h>
57 #ifdef RTE_LIB_METRICS
58 #include <rte_metrics.h>
60 #ifdef RTE_LIB_BITRATESTATS
61 #include <rte_bitrate.h>
63 #ifdef RTE_LIB_LATENCYSTATS
64 #include <rte_latencystats.h>
66 #ifdef RTE_EXEC_ENV_WINDOWS
70 #include <rte_eth_bond.h>
76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
77 #define HUGE_FLAG (0x40000)
79 #define HUGE_FLAG MAP_HUGETLB
82 #ifndef MAP_HUGE_SHIFT
83 /* older kernels (or FreeBSD) will not have this define */
84 #define HUGE_SHIFT (26)
86 #define HUGE_SHIFT MAP_HUGE_SHIFT
89 #define EXTMEM_HEAP_NAME "extmem"
91 * Zone size with the malloc overhead (max of debug and release variants)
92 * must fit into the smallest supported hugepage size (2M),
93 * so that an IOVA-contiguous zone of this size can always be allocated
94 * if there are free 2M hugepages.
96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE)
98 uint16_t verbose_level = 0; /**< Silent by default. */
99 int testpmd_logtype; /**< Log type for testpmd logs */
101 /* use main core for command line ? */
102 uint8_t interactive = 0;
103 uint8_t auto_start = 0;
105 char cmdline_filename[PATH_MAX] = {0};
108 * NUMA support configuration.
109 * When set, the NUMA support attempts to dispatch the allocation of the
110 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
111 * probed ports among the CPU sockets 0 and 1.
112 * Otherwise, all memory is allocated from CPU socket 0.
114 uint8_t numa_support = 1; /**< numa enabled by default */
117 * In UMA mode,all memory is allocated from socket 0 if --socket-num is
120 uint8_t socket_num = UMA_NO_CONFIG;
123 * Select mempool allocation type:
124 * - native: use regular DPDK memory
125 * - anon: use regular DPDK memory to create mempool, but populate using
126 * anonymous memory (may not be IOVA-contiguous)
127 * - xmem: use externally allocated hugepage memory
129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
132 * Store specified sockets on which memory pool to be used by ports
135 uint8_t port_numa[RTE_MAX_ETHPORTS];
138 * Store specified sockets on which RX ring to be used by ports
141 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
144 * Store specified sockets on which TX ring to be used by ports
147 uint8_t txring_numa[RTE_MAX_ETHPORTS];
150 * Record the Ethernet address of peer target ports to which packets are
152 * Must be instantiated with the ethernet addresses of peer traffic generator
155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
156 portid_t nb_peer_eth_addrs = 0;
159 * Probed Target Environment.
161 struct rte_port *ports; /**< For all probed ethernet ports. */
162 portid_t nb_ports; /**< Number of probed ethernet ports. */
163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
164 lcoreid_t nb_lcores; /**< Number of probed logical cores. */
166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
169 * Test Forwarding Configuration.
170 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
171 * nb_fwd_ports <= nb_cfg_ports <= nb_ports
173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
175 portid_t nb_cfg_ports; /**< Number of configured ports. */
176 portid_t nb_fwd_ports; /**< Number of forwarding ports. */
178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */
181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
182 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */
185 * Forwarding engines.
187 struct fwd_engine * fwd_engines[] = {
197 &five_tuple_swap_fwd_engine,
198 #ifdef RTE_LIBRTE_IEEE1588
199 &ieee1588_fwd_engine,
205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
206 uint16_t mempool_flags;
208 struct fwd_config cur_fwd_config;
209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
210 uint32_t retry_enabled;
211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
216 DEFAULT_MBUF_DATA_SIZE
217 }; /**< Mbuf data space size. */
218 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if
219 * specified on command-line. */
220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
222 /** Extended statistics to show. */
223 struct rte_eth_xstat_name *xstats_display;
225 unsigned int xstats_display_num; /**< Size of extended statistics to show */
228 * In container, it cannot terminate the process which running with 'stats-period'
229 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
234 * Max Rx frame size, set by '--max-pkt-len' parameter.
236 uint32_t max_rx_pkt_len;
239 * Configuration of packet segments used to scatter received packets
240 * if some of split features is configured.
242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
243 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */
244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
245 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */
248 * Configuration of packet segments used by the "txonly" processing engine.
250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
252 TXONLY_DEF_PACKET_LEN,
254 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
257 /**< Split policy for packets to TX. */
259 uint8_t txonly_multi_flow;
260 /**< Whether multiple flows are generated in TXONLY mode. */
262 uint32_t tx_pkt_times_inter;
263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
265 uint32_t tx_pkt_times_intra;
266 /**< Timings for send scheduling in TXONLY mode, time between packets. */
268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
273 /* current configuration is in DCB or not,0 means it is not in DCB mode */
274 uint8_t dcb_config = 0;
277 * Configurable number of RX/TX queues.
279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
284 * Configurable number of RX/TX ring descriptors.
285 * Defaults are supplied by drivers via ethdev.
287 #define RTE_TEST_RX_DESC_DEFAULT 0
288 #define RTE_TEST_TX_DESC_DEFAULT 0
289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
292 #define RTE_PMD_PARAM_UNSET -1
294 * Configurable values of RX and TX ring threshold registers.
297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
306 * Configurable value of RX free threshold.
308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
311 * Configurable value of RX drop enable.
313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
316 * Configurable value of TX free threshold.
318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
321 * Configurable value of TX RS bit threshold.
323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
326 * Configurable value of buffered packets before sending.
328 uint16_t noisy_tx_sw_bufsz;
331 * Configurable value of packet buffer timeout.
333 uint16_t noisy_tx_sw_buf_flush_time;
336 * Configurable value for size of VNF internal memory area
337 * used for simulating noisy neighbour behaviour
339 uint64_t noisy_lkup_mem_sz;
342 * Configurable value of number of random writes done in
343 * VNF simulation memory area.
345 uint64_t noisy_lkup_num_writes;
348 * Configurable value of number of random reads done in
349 * VNF simulation memory area.
351 uint64_t noisy_lkup_num_reads;
354 * Configurable value of number of random reads/writes done in
355 * VNF simulation memory area.
357 uint64_t noisy_lkup_num_reads_writes;
360 * Receive Side Scaling (RSS) configuration.
362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */
365 * Port topology configuration
367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
370 * Avoids to flush all the RX streams before starts forwarding.
372 uint8_t no_flush_rx = 0; /* flush by default */
375 * Flow API isolated mode.
377 uint8_t flow_isolate_all;
380 * Avoids to check link status when starting/stopping a port.
382 uint8_t no_link_check = 0; /* check by default */
385 * Don't automatically start all ports in interactive mode.
387 uint8_t no_device_start = 0;
390 * Enable link status change notification
392 uint8_t lsc_interrupt = 1; /* enabled by default */
395 * Enable device removal notification.
397 uint8_t rmv_interrupt = 1; /* enabled by default */
399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
401 /* After attach, port setup is called on event or by iterator */
402 bool setup_on_probe_event = true;
404 /* Clear ptypes on port initialization. */
405 uint8_t clear_ptypes = true;
407 /* Hairpin ports configuration mode. */
408 uint16_t hairpin_mode;
410 /* Pretty printing of ethdev events */
411 static const char * const eth_event_desc[] = {
412 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
413 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
414 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
415 [RTE_ETH_EVENT_INTR_RESET] = "reset",
416 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
417 [RTE_ETH_EVENT_IPSEC] = "IPsec",
418 [RTE_ETH_EVENT_MACSEC] = "MACsec",
419 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
420 [RTE_ETH_EVENT_NEW] = "device probed",
421 [RTE_ETH_EVENT_DESTROY] = "device released",
422 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
423 [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached",
424 [RTE_ETH_EVENT_MAX] = NULL,
428 * Display or mask ether events
429 * Default to all events except VF_MBOX
431 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
432 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
433 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
434 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
435 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
436 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
437 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
438 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
440 * Decide if all memory are locked for performance.
445 * NIC bypass mode configuration options.
448 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
449 /* The NIC bypass watchdog timeout. */
450 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
454 #ifdef RTE_LIB_LATENCYSTATS
457 * Set when latency stats is enabled in the commandline
459 uint8_t latencystats_enabled;
462 * Lcore ID to service latency statistics.
464 lcoreid_t latencystats_lcore_id = -1;
469 * Ethernet device configuration.
471 struct rte_eth_rxmode rx_mode;
473 struct rte_eth_txmode tx_mode = {
474 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
477 struct rte_eth_fdir_conf fdir_conf = {
478 .mode = RTE_FDIR_MODE_NONE,
479 .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
480 .status = RTE_FDIR_REPORT_STATUS,
482 .vlan_tci_mask = 0xFFEF,
484 .src_ip = 0xFFFFFFFF,
485 .dst_ip = 0xFFFFFFFF,
488 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
489 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
491 .src_port_mask = 0xFFFF,
492 .dst_port_mask = 0xFFFF,
493 .mac_addr_byte_mask = 0xFF,
494 .tunnel_type_mask = 1,
495 .tunnel_id_mask = 0xFFFFFFFF,
500 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
503 * Display zero values by default for xstats
505 uint8_t xstats_hide_zero;
508 * Measure of CPU cycles disabled by default
510 uint8_t record_core_cycles;
513 * Display of RX and TX bursts disabled by default
515 uint8_t record_burst_stats;
518 * Number of ports per shared Rx queue group, 0 disable.
522 unsigned int num_sockets = 0;
523 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
525 #ifdef RTE_LIB_BITRATESTATS
526 /* Bitrate statistics */
527 struct rte_stats_bitrates *bitrate_data;
528 lcoreid_t bitrate_lcore_id;
529 uint8_t bitrate_enabled;
533 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
534 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
538 * hexadecimal bitmask of RX mq mode can be enabled.
540 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
543 * Used to set forced link speed
545 uint32_t eth_link_speed;
548 * ID of the current process in multi-process, used to
549 * configure the queues to be polled.
554 * Number of processes in multi-process, used to
555 * configure the queues to be polled.
557 unsigned int num_procs = 1;
560 eth_rx_metadata_negotiate_mp(uint16_t port_id)
562 uint64_t rx_meta_features = 0;
565 if (!is_proc_primary())
568 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
569 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
570 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
572 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
574 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
575 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
579 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
580 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
584 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
585 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
588 } else if (ret != -ENOTSUP) {
589 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
590 port_id, rte_strerror(-ret));
595 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
596 const struct rte_eth_conf *dev_conf)
598 if (is_proc_primary())
599 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
605 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop)
609 portid_t slave_pids[RTE_MAX_ETHPORTS];
610 struct rte_port *port;
615 num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids,
617 if (num_slaves < 0) {
618 fprintf(stderr, "Failed to get slave list for port = %u\n",
623 for (i = 0; i < num_slaves; i++) {
624 slave_pid = slave_pids[i];
625 port = &ports[slave_pid];
627 is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED;
630 RTE_SET_USED(bond_pid);
631 RTE_SET_USED(is_stop);
637 eth_dev_start_mp(uint16_t port_id)
641 if (is_proc_primary()) {
642 ret = rte_eth_dev_start(port_id);
646 struct rte_port *port = &ports[port_id];
649 * Starting a bonded port also starts all slaves under the bonded
650 * device. So if this port is bond device, we need to modify the
651 * port status of these slaves.
653 if (port->bond_flag == 1)
654 return change_bonding_slave_port_status(port_id, false);
661 eth_dev_stop_mp(uint16_t port_id)
665 if (is_proc_primary()) {
666 ret = rte_eth_dev_stop(port_id);
670 struct rte_port *port = &ports[port_id];
673 * Stopping a bonded port also stops all slaves under the bonded
674 * device. So if this port is bond device, we need to modify the
675 * port status of these slaves.
677 if (port->bond_flag == 1)
678 return change_bonding_slave_port_status(port_id, true);
685 mempool_free_mp(struct rte_mempool *mp)
687 if (is_proc_primary())
688 rte_mempool_free(mp);
692 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
694 if (is_proc_primary())
695 return rte_eth_dev_set_mtu(port_id, mtu);
700 /* Forward function declarations */
701 static void setup_attached_port(portid_t pi);
702 static void check_all_ports_link_status(uint32_t port_mask);
703 static int eth_event_callback(portid_t port_id,
704 enum rte_eth_event_type type,
705 void *param, void *ret_param);
706 static void dev_event_callback(const char *device_name,
707 enum rte_dev_event_type type,
709 static void fill_xstats_display_info(void);
712 * Check if all the ports are started.
713 * If yes, return positive value. If not, return zero.
715 static int all_ports_started(void);
718 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
719 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
722 /* Holds the registered mbuf dynamic flags names. */
723 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
727 * Helper function to check if socket is already discovered.
728 * If yes, return positive value. If not, return zero.
731 new_socket_id(unsigned int socket_id)
735 for (i = 0; i < num_sockets; i++) {
736 if (socket_ids[i] == socket_id)
743 * Setup default configuration.
746 set_default_fwd_lcores_config(void)
750 unsigned int sock_num;
753 for (i = 0; i < RTE_MAX_LCORE; i++) {
754 if (!rte_lcore_is_enabled(i))
756 sock_num = rte_lcore_to_socket_id(i);
757 if (new_socket_id(sock_num)) {
758 if (num_sockets >= RTE_MAX_NUMA_NODES) {
759 rte_exit(EXIT_FAILURE,
760 "Total sockets greater than %u\n",
763 socket_ids[num_sockets++] = sock_num;
765 if (i == rte_get_main_lcore())
767 fwd_lcores_cpuids[nb_lc++] = i;
769 nb_lcores = (lcoreid_t) nb_lc;
770 nb_cfg_lcores = nb_lcores;
775 set_def_peer_eth_addrs(void)
779 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
780 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
781 peer_eth_addrs[i].addr_bytes[5] = i;
786 set_default_fwd_ports_config(void)
791 RTE_ETH_FOREACH_DEV(pt_id) {
792 fwd_ports_ids[i++] = pt_id;
794 /* Update sockets info according to the attached device */
795 int socket_id = rte_eth_dev_socket_id(pt_id);
796 if (socket_id >= 0 && new_socket_id(socket_id)) {
797 if (num_sockets >= RTE_MAX_NUMA_NODES) {
798 rte_exit(EXIT_FAILURE,
799 "Total sockets greater than %u\n",
802 socket_ids[num_sockets++] = socket_id;
806 nb_cfg_ports = nb_ports;
807 nb_fwd_ports = nb_ports;
811 set_def_fwd_config(void)
813 set_default_fwd_lcores_config();
814 set_def_peer_eth_addrs();
815 set_default_fwd_ports_config();
818 #ifndef RTE_EXEC_ENV_WINDOWS
819 /* extremely pessimistic estimation of memory required to create a mempool */
821 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
823 unsigned int n_pages, mbuf_per_pg, leftover;
824 uint64_t total_mem, mbuf_mem, obj_sz;
826 /* there is no good way to predict how much space the mempool will
827 * occupy because it will allocate chunks on the fly, and some of those
828 * will come from default DPDK memory while some will come from our
829 * external memory, so just assume 128MB will be enough for everyone.
831 uint64_t hdr_mem = 128 << 20;
833 /* account for possible non-contiguousness */
834 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
836 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
840 mbuf_per_pg = pgsz / obj_sz;
841 leftover = (nb_mbufs % mbuf_per_pg) > 0;
842 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
844 mbuf_mem = n_pages * pgsz;
846 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
848 if (total_mem > SIZE_MAX) {
849 TESTPMD_LOG(ERR, "Memory size too big\n");
852 *out = (size_t)total_mem;
858 pagesz_flags(uint64_t page_sz)
860 /* as per mmap() manpage, all page sizes are log2 of page size
861 * shifted by MAP_HUGE_SHIFT
863 int log2 = rte_log2_u64(page_sz);
865 return (log2 << HUGE_SHIFT);
869 alloc_mem(size_t memsz, size_t pgsz, bool huge)
874 /* allocate anonymous hugepages */
875 flags = MAP_ANONYMOUS | MAP_PRIVATE;
877 flags |= HUGE_FLAG | pagesz_flags(pgsz);
879 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
880 if (addr == MAP_FAILED)
886 struct extmem_param {
890 rte_iova_t *iova_table;
891 unsigned int iova_table_len;
895 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
898 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
899 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */
900 unsigned int cur_page, n_pages, pgsz_idx;
901 size_t mem_sz, cur_pgsz;
902 rte_iova_t *iovas = NULL;
906 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
907 /* skip anything that is too big */
908 if (pgsizes[pgsz_idx] > SIZE_MAX)
911 cur_pgsz = pgsizes[pgsz_idx];
913 /* if we were told not to allocate hugepages, override */
915 cur_pgsz = sysconf(_SC_PAGESIZE);
917 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
919 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
923 /* allocate our memory */
924 addr = alloc_mem(mem_sz, cur_pgsz, huge);
926 /* if we couldn't allocate memory with a specified page size,
927 * that doesn't mean we can't do it with other page sizes, so
933 /* store IOVA addresses for every page in this memory area */
934 n_pages = mem_sz / cur_pgsz;
936 iovas = malloc(sizeof(*iovas) * n_pages);
939 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
942 /* lock memory if it's not huge pages */
946 /* populate IOVA addresses */
947 for (cur_page = 0; cur_page < n_pages; cur_page++) {
952 offset = cur_pgsz * cur_page;
953 cur = RTE_PTR_ADD(addr, offset);
955 /* touch the page before getting its IOVA */
956 *(volatile char *)cur = 0;
958 iova = rte_mem_virt2iova(cur);
960 iovas[cur_page] = iova;
965 /* if we couldn't allocate anything */
971 param->pgsz = cur_pgsz;
972 param->iova_table = iovas;
973 param->iova_table_len = n_pages;
979 munmap(addr, mem_sz);
985 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
987 struct extmem_param param;
990 memset(¶m, 0, sizeof(param));
992 /* check if our heap exists */
993 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
995 /* create our heap */
996 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
998 TESTPMD_LOG(ERR, "Cannot create heap\n");
1003 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
1005 TESTPMD_LOG(ERR, "Cannot create memory area\n");
1009 /* we now have a valid memory area, so add it to heap */
1010 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
1011 param.addr, param.len, param.iova_table,
1012 param.iova_table_len, param.pgsz);
1014 /* when using VFIO, memory is automatically mapped for DMA by EAL */
1016 /* not needed any more */
1017 free(param.iova_table);
1020 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
1021 munmap(param.addr, param.len);
1027 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
1033 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1034 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1039 RTE_ETH_FOREACH_DEV(pid) {
1040 struct rte_eth_dev_info dev_info;
1042 ret = eth_dev_info_get_print_err(pid, &dev_info);
1045 "unable to get device info for port %d on addr 0x%p,"
1046 "mempool unmapping will not be performed\n",
1051 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
1054 "unable to DMA unmap addr 0x%p "
1056 memhdr->addr, dev_info.device->name);
1059 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
1062 "unable to un-register addr 0x%p\n", memhdr->addr);
1067 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1068 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1071 size_t page_size = sysconf(_SC_PAGESIZE);
1074 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1078 "unable to register addr 0x%p\n", memhdr->addr);
1081 RTE_ETH_FOREACH_DEV(pid) {
1082 struct rte_eth_dev_info dev_info;
1084 ret = eth_dev_info_get_print_err(pid, &dev_info);
1087 "unable to get device info for port %d on addr 0x%p,"
1088 "mempool mapping will not be performed\n",
1092 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1095 "unable to DMA map addr 0x%p "
1097 memhdr->addr, dev_info.device->name);
1104 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1105 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1107 struct rte_pktmbuf_extmem *xmem;
1108 unsigned int ext_num, zone_num, elt_num;
1111 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1112 elt_num = EXTBUF_ZONE_SIZE / elt_size;
1113 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1115 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1117 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1118 "external buffer descriptors\n");
1122 for (ext_num = 0; ext_num < zone_num; ext_num++) {
1123 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1124 const struct rte_memzone *mz;
1125 char mz_name[RTE_MEMZONE_NAMESIZE];
1128 ret = snprintf(mz_name, sizeof(mz_name),
1129 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1130 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1131 errno = ENAMETOOLONG;
1135 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE,
1137 RTE_MEMZONE_IOVA_CONTIG |
1139 RTE_MEMZONE_SIZE_HINT_ONLY);
1142 * The caller exits on external buffer creation
1143 * error, so there is no need to free memzones.
1149 xseg->buf_ptr = mz->addr;
1150 xseg->buf_iova = mz->iova;
1151 xseg->buf_len = EXTBUF_ZONE_SIZE;
1152 xseg->elt_size = elt_size;
1154 if (ext_num == 0 && xmem != NULL) {
1163 * Configuration initialisation done once at init time.
1165 static struct rte_mempool *
1166 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1167 unsigned int socket_id, uint16_t size_idx)
1169 char pool_name[RTE_MEMPOOL_NAMESIZE];
1170 struct rte_mempool *rte_mp = NULL;
1171 #ifndef RTE_EXEC_ENV_WINDOWS
1174 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1176 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1177 if (!is_proc_primary()) {
1178 rte_mp = rte_mempool_lookup(pool_name);
1180 rte_exit(EXIT_FAILURE,
1181 "Get mbuf pool for socket %u failed: %s\n",
1182 socket_id, rte_strerror(rte_errno));
1187 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1188 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1190 switch (mp_alloc_type) {
1191 case MP_ALLOC_NATIVE:
1193 /* wrapper to rte_mempool_create() */
1194 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1195 rte_mbuf_best_mempool_ops());
1196 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1197 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1200 #ifndef RTE_EXEC_ENV_WINDOWS
1203 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1204 mb_size, (unsigned int) mb_mempool_cache,
1205 sizeof(struct rte_pktmbuf_pool_private),
1206 socket_id, mempool_flags);
1210 if (rte_mempool_populate_anon(rte_mp) == 0) {
1211 rte_mempool_free(rte_mp);
1215 rte_pktmbuf_pool_init(rte_mp, NULL);
1216 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1217 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1221 case MP_ALLOC_XMEM_HUGE:
1224 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1226 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1227 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1230 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1231 if (heap_socket < 0)
1232 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1234 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1235 rte_mbuf_best_mempool_ops());
1236 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1237 mb_mempool_cache, 0, mbuf_seg_size,
1244 struct rte_pktmbuf_extmem *ext_mem;
1245 unsigned int ext_num;
1247 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1248 socket_id, pool_name, &ext_mem);
1250 rte_exit(EXIT_FAILURE,
1251 "Can't create pinned data buffers\n");
1253 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1254 rte_mbuf_best_mempool_ops());
1255 rte_mp = rte_pktmbuf_pool_create_extbuf
1256 (pool_name, nb_mbuf, mb_mempool_cache,
1257 0, mbuf_seg_size, socket_id,
1264 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1268 #ifndef RTE_EXEC_ENV_WINDOWS
1271 if (rte_mp == NULL) {
1272 rte_exit(EXIT_FAILURE,
1273 "Creation of mbuf pool for socket %u failed: %s\n",
1274 socket_id, rte_strerror(rte_errno));
1275 } else if (verbose_level > 0) {
1276 rte_mempool_dump(stdout, rte_mp);
1282 * Check given socket id is valid or not with NUMA mode,
1283 * if valid, return 0, else return -1
1286 check_socket_id(const unsigned int socket_id)
1288 static int warning_once = 0;
1290 if (new_socket_id(socket_id)) {
1291 if (!warning_once && numa_support)
1293 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1301 * Get the allowed maximum number of RX queues.
1302 * *pid return the port id which has minimal value of
1303 * max_rx_queues in all ports.
1306 get_allowed_max_nb_rxq(portid_t *pid)
1308 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1309 bool max_rxq_valid = false;
1311 struct rte_eth_dev_info dev_info;
1313 RTE_ETH_FOREACH_DEV(pi) {
1314 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1317 max_rxq_valid = true;
1318 if (dev_info.max_rx_queues < allowed_max_rxq) {
1319 allowed_max_rxq = dev_info.max_rx_queues;
1323 return max_rxq_valid ? allowed_max_rxq : 0;
1327 * Check input rxq is valid or not.
1328 * If input rxq is not greater than any of maximum number
1329 * of RX queues of all ports, it is valid.
1330 * if valid, return 0, else return -1
1333 check_nb_rxq(queueid_t rxq)
1335 queueid_t allowed_max_rxq;
1338 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1339 if (rxq > allowed_max_rxq) {
1341 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1342 rxq, allowed_max_rxq, pid);
1349 * Get the allowed maximum number of TX queues.
1350 * *pid return the port id which has minimal value of
1351 * max_tx_queues in all ports.
1354 get_allowed_max_nb_txq(portid_t *pid)
1356 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1357 bool max_txq_valid = false;
1359 struct rte_eth_dev_info dev_info;
1361 RTE_ETH_FOREACH_DEV(pi) {
1362 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1365 max_txq_valid = true;
1366 if (dev_info.max_tx_queues < allowed_max_txq) {
1367 allowed_max_txq = dev_info.max_tx_queues;
1371 return max_txq_valid ? allowed_max_txq : 0;
1375 * Check input txq is valid or not.
1376 * If input txq is not greater than any of maximum number
1377 * of TX queues of all ports, it is valid.
1378 * if valid, return 0, else return -1
1381 check_nb_txq(queueid_t txq)
1383 queueid_t allowed_max_txq;
1386 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1387 if (txq > allowed_max_txq) {
1389 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1390 txq, allowed_max_txq, pid);
1397 * Get the allowed maximum number of RXDs of every rx queue.
1398 * *pid return the port id which has minimal value of
1399 * max_rxd in all queues of all ports.
1402 get_allowed_max_nb_rxd(portid_t *pid)
1404 uint16_t allowed_max_rxd = UINT16_MAX;
1406 struct rte_eth_dev_info dev_info;
1408 RTE_ETH_FOREACH_DEV(pi) {
1409 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1412 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1413 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1417 return allowed_max_rxd;
1421 * Get the allowed minimal number of RXDs of every rx queue.
1422 * *pid return the port id which has minimal value of
1423 * min_rxd in all queues of all ports.
1426 get_allowed_min_nb_rxd(portid_t *pid)
1428 uint16_t allowed_min_rxd = 0;
1430 struct rte_eth_dev_info dev_info;
1432 RTE_ETH_FOREACH_DEV(pi) {
1433 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1436 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1437 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1442 return allowed_min_rxd;
1446 * Check input rxd is valid or not.
1447 * If input rxd is not greater than any of maximum number
1448 * of RXDs of every Rx queues and is not less than any of
1449 * minimal number of RXDs of every Rx queues, it is valid.
1450 * if valid, return 0, else return -1
1453 check_nb_rxd(queueid_t rxd)
1455 uint16_t allowed_max_rxd;
1456 uint16_t allowed_min_rxd;
1459 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1460 if (rxd > allowed_max_rxd) {
1462 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1463 rxd, allowed_max_rxd, pid);
1467 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1468 if (rxd < allowed_min_rxd) {
1470 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1471 rxd, allowed_min_rxd, pid);
1479 * Get the allowed maximum number of TXDs of every rx queues.
1480 * *pid return the port id which has minimal value of
1481 * max_txd in every tx queue.
1484 get_allowed_max_nb_txd(portid_t *pid)
1486 uint16_t allowed_max_txd = UINT16_MAX;
1488 struct rte_eth_dev_info dev_info;
1490 RTE_ETH_FOREACH_DEV(pi) {
1491 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1494 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1495 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1499 return allowed_max_txd;
1503 * Get the allowed maximum number of TXDs of every tx queues.
1504 * *pid return the port id which has minimal value of
1505 * min_txd in every tx queue.
1508 get_allowed_min_nb_txd(portid_t *pid)
1510 uint16_t allowed_min_txd = 0;
1512 struct rte_eth_dev_info dev_info;
1514 RTE_ETH_FOREACH_DEV(pi) {
1515 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1518 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1519 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1524 return allowed_min_txd;
1528 * Check input txd is valid or not.
1529 * If input txd is not greater than any of maximum number
1530 * of TXDs of every Rx queues, it is valid.
1531 * if valid, return 0, else return -1
1534 check_nb_txd(queueid_t txd)
1536 uint16_t allowed_max_txd;
1537 uint16_t allowed_min_txd;
1540 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1541 if (txd > allowed_max_txd) {
1543 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1544 txd, allowed_max_txd, pid);
1548 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1549 if (txd < allowed_min_txd) {
1551 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1552 txd, allowed_min_txd, pid);
1560 * Get the allowed maximum number of hairpin queues.
1561 * *pid return the port id which has minimal value of
1562 * max_hairpin_queues in all ports.
1565 get_allowed_max_nb_hairpinq(portid_t *pid)
1567 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1569 struct rte_eth_hairpin_cap cap;
1571 RTE_ETH_FOREACH_DEV(pi) {
1572 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1576 if (cap.max_nb_queues < allowed_max_hairpinq) {
1577 allowed_max_hairpinq = cap.max_nb_queues;
1581 return allowed_max_hairpinq;
1585 * Check input hairpin is valid or not.
1586 * If input hairpin is not greater than any of maximum number
1587 * of hairpin queues of all ports, it is valid.
1588 * if valid, return 0, else return -1
1591 check_nb_hairpinq(queueid_t hairpinq)
1593 queueid_t allowed_max_hairpinq;
1596 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1597 if (hairpinq > allowed_max_hairpinq) {
1599 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1600 hairpinq, allowed_max_hairpinq, pid);
1607 get_eth_overhead(struct rte_eth_dev_info *dev_info)
1609 uint32_t eth_overhead;
1611 if (dev_info->max_mtu != UINT16_MAX &&
1612 dev_info->max_rx_pktlen > dev_info->max_mtu)
1613 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1615 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1617 return eth_overhead;
1621 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1623 struct rte_port *port = &ports[pid];
1627 eth_rx_metadata_negotiate_mp(pid);
1629 port->dev_conf.txmode = tx_mode;
1630 port->dev_conf.rxmode = rx_mode;
1632 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1634 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1636 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1637 port->dev_conf.txmode.offloads &=
1638 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1640 /* Apply Rx offloads configuration */
1641 for (i = 0; i < port->dev_info.max_rx_queues; i++)
1642 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads;
1643 /* Apply Tx offloads configuration */
1644 for (i = 0; i < port->dev_info.max_tx_queues; i++)
1645 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads;
1648 port->dev_conf.link_speeds = eth_link_speed;
1651 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1652 get_eth_overhead(&port->dev_info);
1654 /* set flag to initialize port/queue */
1655 port->need_reconfig = 1;
1656 port->need_reconfig_queues = 1;
1657 port->socket_id = socket_id;
1658 port->tx_metadata = 0;
1661 * Check for maximum number of segments per MTU.
1662 * Accordingly update the mbuf data size.
1664 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1665 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1666 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1669 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1670 uint16_t data_size = (mtu + eth_overhead) /
1671 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1672 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1674 if (buffer_size > mbuf_data_size[0]) {
1675 mbuf_data_size[0] = buffer_size;
1676 TESTPMD_LOG(WARNING,
1677 "Configured mbuf size of the first segment %hu\n",
1688 struct rte_mempool *mbp;
1689 unsigned int nb_mbuf_per_pool;
1692 struct rte_gro_param gro_param;
1698 /* Configuration of logical cores. */
1699 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1700 sizeof(struct fwd_lcore *) * nb_lcores,
1701 RTE_CACHE_LINE_SIZE);
1702 if (fwd_lcores == NULL) {
1703 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1704 "failed\n", nb_lcores);
1706 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1707 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1708 sizeof(struct fwd_lcore),
1709 RTE_CACHE_LINE_SIZE);
1710 if (fwd_lcores[lc_id] == NULL) {
1711 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1714 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1717 RTE_ETH_FOREACH_DEV(pid) {
1721 socket_id = port_numa[pid];
1722 if (port_numa[pid] == NUMA_NO_CONFIG) {
1723 socket_id = rte_eth_dev_socket_id(pid);
1726 * if socket_id is invalid,
1727 * set to the first available socket.
1729 if (check_socket_id(socket_id) < 0)
1730 socket_id = socket_ids[0];
1733 socket_id = (socket_num == UMA_NO_CONFIG) ?
1736 /* Apply default TxRx configuration for all ports */
1737 init_config_port_offloads(pid, socket_id);
1740 * Create pools of mbuf.
1741 * If NUMA support is disabled, create a single pool of mbuf in
1742 * socket 0 memory by default.
1743 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1745 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1746 * nb_txd can be configured at run time.
1748 if (param_total_num_mbufs)
1749 nb_mbuf_per_pool = param_total_num_mbufs;
1751 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1752 (nb_lcores * mb_mempool_cache) +
1753 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1754 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1760 for (i = 0; i < num_sockets; i++)
1761 for (j = 0; j < mbuf_data_size_n; j++)
1762 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1763 mbuf_pool_create(mbuf_data_size[j],
1769 for (i = 0; i < mbuf_data_size_n; i++)
1770 mempools[i] = mbuf_pool_create
1773 socket_num == UMA_NO_CONFIG ?
1780 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1781 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1784 * Records which Mbuf pool to use by each logical core, if needed.
1786 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1787 mbp = mbuf_pool_find(
1788 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1791 mbp = mbuf_pool_find(0, 0);
1792 fwd_lcores[lc_id]->mbp = mbp;
1794 /* initialize GSO context */
1795 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1796 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1797 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1798 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1800 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1807 /* create a gro context for each lcore */
1808 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1809 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1810 gro_param.max_item_per_flow = MAX_PKT_BURST;
1811 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1812 gro_param.socket_id = rte_lcore_to_socket_id(
1813 fwd_lcores_cpuids[lc_id]);
1814 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1815 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1816 rte_exit(EXIT_FAILURE,
1817 "rte_gro_ctx_create() failed\n");
1825 reconfig(portid_t new_port_id, unsigned socket_id)
1827 /* Reconfiguration of Ethernet ports. */
1828 init_config_port_offloads(new_port_id, socket_id);
1833 init_fwd_streams(void)
1836 struct rte_port *port;
1837 streamid_t sm_id, nb_fwd_streams_new;
1840 /* set socket id according to numa or not */
1841 RTE_ETH_FOREACH_DEV(pid) {
1843 if (nb_rxq > port->dev_info.max_rx_queues) {
1845 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1846 nb_rxq, port->dev_info.max_rx_queues);
1849 if (nb_txq > port->dev_info.max_tx_queues) {
1851 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1852 nb_txq, port->dev_info.max_tx_queues);
1856 if (port_numa[pid] != NUMA_NO_CONFIG)
1857 port->socket_id = port_numa[pid];
1859 port->socket_id = rte_eth_dev_socket_id(pid);
1862 * if socket_id is invalid,
1863 * set to the first available socket.
1865 if (check_socket_id(port->socket_id) < 0)
1866 port->socket_id = socket_ids[0];
1870 if (socket_num == UMA_NO_CONFIG)
1871 port->socket_id = 0;
1873 port->socket_id = socket_num;
1877 q = RTE_MAX(nb_rxq, nb_txq);
1880 "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1883 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1884 if (nb_fwd_streams_new == nb_fwd_streams)
1887 if (fwd_streams != NULL) {
1888 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1889 if (fwd_streams[sm_id] == NULL)
1891 rte_free(fwd_streams[sm_id]);
1892 fwd_streams[sm_id] = NULL;
1894 rte_free(fwd_streams);
1899 nb_fwd_streams = nb_fwd_streams_new;
1900 if (nb_fwd_streams) {
1901 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1902 sizeof(struct fwd_stream *) * nb_fwd_streams,
1903 RTE_CACHE_LINE_SIZE);
1904 if (fwd_streams == NULL)
1905 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1906 " (struct fwd_stream *)) failed\n",
1909 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1910 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1911 " struct fwd_stream", sizeof(struct fwd_stream),
1912 RTE_CACHE_LINE_SIZE);
1913 if (fwd_streams[sm_id] == NULL)
1914 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1915 "(struct fwd_stream) failed\n");
1923 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1925 uint64_t total_burst, sburst;
1927 uint64_t burst_stats[4];
1928 uint16_t pktnb_stats[4];
1930 int burst_percent[4], sburstp;
1934 * First compute the total number of packet bursts and the
1935 * two highest numbers of bursts of the same number of packets.
1937 memset(&burst_stats, 0x0, sizeof(burst_stats));
1938 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1940 /* Show stats for 0 burst size always */
1941 total_burst = pbs->pkt_burst_spread[0];
1942 burst_stats[0] = pbs->pkt_burst_spread[0];
1945 /* Find the next 2 burst sizes with highest occurrences. */
1946 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1947 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1952 total_burst += nb_burst;
1954 if (nb_burst > burst_stats[1]) {
1955 burst_stats[2] = burst_stats[1];
1956 pktnb_stats[2] = pktnb_stats[1];
1957 burst_stats[1] = nb_burst;
1958 pktnb_stats[1] = nb_pkt;
1959 } else if (nb_burst > burst_stats[2]) {
1960 burst_stats[2] = nb_burst;
1961 pktnb_stats[2] = nb_pkt;
1964 if (total_burst == 0)
1967 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1968 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1970 printf("%d%% of other]\n", 100 - sburstp);
1974 sburst += burst_stats[i];
1975 if (sburst == total_burst) {
1976 printf("%d%% of %d pkts]\n",
1977 100 - sburstp, (int) pktnb_stats[i]);
1982 (double)burst_stats[i] / total_burst * 100;
1983 printf("%d%% of %d pkts + ",
1984 burst_percent[i], (int) pktnb_stats[i]);
1985 sburstp += burst_percent[i];
1990 fwd_stream_stats_display(streamid_t stream_id)
1992 struct fwd_stream *fs;
1993 static const char *fwd_top_stats_border = "-------";
1995 fs = fwd_streams[stream_id];
1996 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1997 (fs->fwd_dropped == 0))
1999 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
2000 "TX Port=%2d/Queue=%2d %s\n",
2001 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
2002 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
2003 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
2004 " TX-dropped: %-14"PRIu64,
2005 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
2007 /* if checksum mode */
2008 if (cur_fwd_eng == &csum_fwd_engine) {
2009 printf(" RX- bad IP checksum: %-14"PRIu64
2010 " Rx- bad L4 checksum: %-14"PRIu64
2011 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
2012 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
2013 fs->rx_bad_outer_l4_csum);
2014 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
2015 fs->rx_bad_outer_ip_csum);
2020 if (record_burst_stats) {
2021 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
2022 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
2027 fwd_stats_display(void)
2029 static const char *fwd_stats_border = "----------------------";
2030 static const char *acc_stats_border = "+++++++++++++++";
2032 struct fwd_stream *rx_stream;
2033 struct fwd_stream *tx_stream;
2034 uint64_t tx_dropped;
2035 uint64_t rx_bad_ip_csum;
2036 uint64_t rx_bad_l4_csum;
2037 uint64_t rx_bad_outer_l4_csum;
2038 uint64_t rx_bad_outer_ip_csum;
2039 } ports_stats[RTE_MAX_ETHPORTS];
2040 uint64_t total_rx_dropped = 0;
2041 uint64_t total_tx_dropped = 0;
2042 uint64_t total_rx_nombuf = 0;
2043 struct rte_eth_stats stats;
2044 uint64_t fwd_cycles = 0;
2045 uint64_t total_recv = 0;
2046 uint64_t total_xmit = 0;
2047 struct rte_port *port;
2053 memset(ports_stats, 0, sizeof(ports_stats));
2055 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2056 struct fwd_stream *fs = fwd_streams[sm_id];
2058 if (cur_fwd_config.nb_fwd_streams >
2059 cur_fwd_config.nb_fwd_ports) {
2060 fwd_stream_stats_display(sm_id);
2062 ports_stats[fs->tx_port].tx_stream = fs;
2063 ports_stats[fs->rx_port].rx_stream = fs;
2066 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
2068 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
2069 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2070 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2071 fs->rx_bad_outer_l4_csum;
2072 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2073 fs->rx_bad_outer_ip_csum;
2075 if (record_core_cycles)
2076 fwd_cycles += fs->core_cycles;
2078 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2079 pt_id = fwd_ports_ids[i];
2080 port = &ports[pt_id];
2082 ret = rte_eth_stats_get(pt_id, &stats);
2085 "%s: Error: failed to get stats (port %u): %d",
2086 __func__, pt_id, ret);
2089 stats.ipackets -= port->stats.ipackets;
2090 stats.opackets -= port->stats.opackets;
2091 stats.ibytes -= port->stats.ibytes;
2092 stats.obytes -= port->stats.obytes;
2093 stats.imissed -= port->stats.imissed;
2094 stats.oerrors -= port->stats.oerrors;
2095 stats.rx_nombuf -= port->stats.rx_nombuf;
2097 total_recv += stats.ipackets;
2098 total_xmit += stats.opackets;
2099 total_rx_dropped += stats.imissed;
2100 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2101 total_tx_dropped += stats.oerrors;
2102 total_rx_nombuf += stats.rx_nombuf;
2104 printf("\n %s Forward statistics for port %-2d %s\n",
2105 fwd_stats_border, pt_id, fwd_stats_border);
2107 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2108 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2109 stats.ipackets + stats.imissed);
2111 if (cur_fwd_eng == &csum_fwd_engine) {
2112 printf(" Bad-ipcsum: %-14"PRIu64
2113 " Bad-l4csum: %-14"PRIu64
2114 "Bad-outer-l4csum: %-14"PRIu64"\n",
2115 ports_stats[pt_id].rx_bad_ip_csum,
2116 ports_stats[pt_id].rx_bad_l4_csum,
2117 ports_stats[pt_id].rx_bad_outer_l4_csum);
2118 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n",
2119 ports_stats[pt_id].rx_bad_outer_ip_csum);
2121 if (stats.ierrors + stats.rx_nombuf > 0) {
2122 printf(" RX-error: %-"PRIu64"\n", stats.ierrors);
2123 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2126 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2127 "TX-total: %-"PRIu64"\n",
2128 stats.opackets, ports_stats[pt_id].tx_dropped,
2129 stats.opackets + ports_stats[pt_id].tx_dropped);
2131 if (record_burst_stats) {
2132 if (ports_stats[pt_id].rx_stream)
2133 pkt_burst_stats_display("RX",
2134 &ports_stats[pt_id].rx_stream->rx_burst_stats);
2135 if (ports_stats[pt_id].tx_stream)
2136 pkt_burst_stats_display("TX",
2137 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2140 printf(" %s--------------------------------%s\n",
2141 fwd_stats_border, fwd_stats_border);
2144 printf("\n %s Accumulated forward statistics for all ports"
2146 acc_stats_border, acc_stats_border);
2147 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2149 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2151 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2152 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2153 if (total_rx_nombuf > 0)
2154 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2155 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
2157 acc_stats_border, acc_stats_border);
2158 if (record_core_cycles) {
2159 #define CYC_PER_MHZ 1E6
2160 if (total_recv > 0 || total_xmit > 0) {
2161 uint64_t total_pkts = 0;
2162 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2163 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2164 total_pkts = total_xmit;
2166 total_pkts = total_recv;
2168 printf("\n CPU cycles/packet=%.2F (total cycles="
2169 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2171 (double) fwd_cycles / total_pkts,
2172 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2173 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2179 fwd_stats_reset(void)
2186 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2187 pt_id = fwd_ports_ids[i];
2188 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2191 "%s: Error: failed to clear stats (port %u):%d",
2192 __func__, pt_id, ret);
2194 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2195 struct fwd_stream *fs = fwd_streams[sm_id];
2199 fs->fwd_dropped = 0;
2200 fs->rx_bad_ip_csum = 0;
2201 fs->rx_bad_l4_csum = 0;
2202 fs->rx_bad_outer_l4_csum = 0;
2203 fs->rx_bad_outer_ip_csum = 0;
2205 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2206 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2207 fs->core_cycles = 0;
2212 flush_fwd_rx_queues(void)
2214 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2221 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2222 uint64_t timer_period;
2224 if (num_procs > 1) {
2225 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2229 /* convert to number of cycles */
2230 timer_period = rte_get_timer_hz(); /* 1 second timeout */
2232 for (j = 0; j < 2; j++) {
2233 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2234 for (rxq = 0; rxq < nb_rxq; rxq++) {
2235 port_id = fwd_ports_ids[rxp];
2237 /* Polling stopped queues is prohibited. */
2238 if (ports[port_id].rxq[rxq].state ==
2239 RTE_ETH_QUEUE_STATE_STOPPED)
2243 * testpmd can stuck in the below do while loop
2244 * if rte_eth_rx_burst() always returns nonzero
2245 * packets. So timer is added to exit this loop
2246 * after 1sec timer expiry.
2248 prev_tsc = rte_rdtsc();
2250 nb_rx = rte_eth_rx_burst(port_id, rxq,
2251 pkts_burst, MAX_PKT_BURST);
2252 for (i = 0; i < nb_rx; i++)
2253 rte_pktmbuf_free(pkts_burst[i]);
2255 cur_tsc = rte_rdtsc();
2256 diff_tsc = cur_tsc - prev_tsc;
2257 timer_tsc += diff_tsc;
2258 } while ((nb_rx > 0) &&
2259 (timer_tsc < timer_period));
2263 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2268 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2270 struct fwd_stream **fsm;
2273 #ifdef RTE_LIB_BITRATESTATS
2274 uint64_t tics_per_1sec;
2275 uint64_t tics_datum;
2276 uint64_t tics_current;
2277 uint16_t i, cnt_ports;
2279 cnt_ports = nb_ports;
2280 tics_datum = rte_rdtsc();
2281 tics_per_1sec = rte_get_timer_hz();
2283 fsm = &fwd_streams[fc->stream_idx];
2284 nb_fs = fc->stream_nb;
2286 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2287 if (!fsm[sm_id]->disabled)
2288 (*pkt_fwd)(fsm[sm_id]);
2289 #ifdef RTE_LIB_BITRATESTATS
2290 if (bitrate_enabled != 0 &&
2291 bitrate_lcore_id == rte_lcore_id()) {
2292 tics_current = rte_rdtsc();
2293 if (tics_current - tics_datum >= tics_per_1sec) {
2294 /* Periodic bitrate calculation */
2295 for (i = 0; i < cnt_ports; i++)
2296 rte_stats_bitrate_calc(bitrate_data,
2298 tics_datum = tics_current;
2302 #ifdef RTE_LIB_LATENCYSTATS
2303 if (latencystats_enabled != 0 &&
2304 latencystats_lcore_id == rte_lcore_id())
2305 rte_latencystats_update();
2308 } while (! fc->stopped);
2312 start_pkt_forward_on_core(void *fwd_arg)
2314 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2315 cur_fwd_config.fwd_eng->packet_fwd);
2320 * Run the TXONLY packet forwarding engine to send a single burst of packets.
2321 * Used to start communication flows in network loopback test configurations.
2324 run_one_txonly_burst_on_core(void *fwd_arg)
2326 struct fwd_lcore *fwd_lc;
2327 struct fwd_lcore tmp_lcore;
2329 fwd_lc = (struct fwd_lcore *) fwd_arg;
2330 tmp_lcore = *fwd_lc;
2331 tmp_lcore.stopped = 1;
2332 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2337 * Launch packet forwarding:
2338 * - Setup per-port forwarding context.
2339 * - launch logical cores with their forwarding configuration.
2342 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2348 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2349 lc_id = fwd_lcores_cpuids[i];
2350 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2351 fwd_lcores[i]->stopped = 0;
2352 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2353 fwd_lcores[i], lc_id);
2356 "launch lcore %u failed - diag=%d\n",
2363 * Launch packet forwarding configuration.
2366 start_packet_forwarding(int with_tx_first)
2368 port_fwd_begin_t port_fwd_begin;
2369 port_fwd_end_t port_fwd_end;
2370 stream_init_t stream_init = cur_fwd_eng->stream_init;
2373 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2374 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2376 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2377 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2379 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2380 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2381 (!nb_rxq || !nb_txq))
2382 rte_exit(EXIT_FAILURE,
2383 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2384 cur_fwd_eng->fwd_mode_name);
2386 if (all_ports_started() == 0) {
2387 fprintf(stderr, "Not all ports were started\n");
2390 if (test_done == 0) {
2391 fprintf(stderr, "Packet forwarding already started\n");
2397 pkt_fwd_config_display(&cur_fwd_config);
2398 if (!pkt_fwd_shared_rxq_check())
2401 if (stream_init != NULL)
2402 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++)
2403 stream_init(fwd_streams[i]);
2405 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2406 if (port_fwd_begin != NULL) {
2407 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2408 if (port_fwd_begin(fwd_ports_ids[i])) {
2410 "Packet forwarding is not ready\n");
2416 if (with_tx_first) {
2417 port_fwd_begin = tx_only_engine.port_fwd_begin;
2418 if (port_fwd_begin != NULL) {
2419 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2420 if (port_fwd_begin(fwd_ports_ids[i])) {
2422 "Packet forwarding is not ready\n");
2432 flush_fwd_rx_queues();
2434 rxtx_config_display();
2437 if (with_tx_first) {
2438 while (with_tx_first--) {
2439 launch_packet_forwarding(
2440 run_one_txonly_burst_on_core);
2441 rte_eal_mp_wait_lcore();
2443 port_fwd_end = tx_only_engine.port_fwd_end;
2444 if (port_fwd_end != NULL) {
2445 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2446 (*port_fwd_end)(fwd_ports_ids[i]);
2449 launch_packet_forwarding(start_pkt_forward_on_core);
2453 stop_packet_forwarding(void)
2455 port_fwd_end_t port_fwd_end;
2461 fprintf(stderr, "Packet forwarding not started\n");
2464 printf("Telling cores to stop...");
2465 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2466 fwd_lcores[lc_id]->stopped = 1;
2467 printf("\nWaiting for lcores to finish...\n");
2468 rte_eal_mp_wait_lcore();
2469 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2470 if (port_fwd_end != NULL) {
2471 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2472 pt_id = fwd_ports_ids[i];
2473 (*port_fwd_end)(pt_id);
2477 fwd_stats_display();
2479 printf("\nDone.\n");
2484 dev_set_link_up(portid_t pid)
2486 if (rte_eth_dev_set_link_up(pid) < 0)
2487 fprintf(stderr, "\nSet link up fail.\n");
2491 dev_set_link_down(portid_t pid)
2493 if (rte_eth_dev_set_link_down(pid) < 0)
2494 fprintf(stderr, "\nSet link down fail.\n");
2498 all_ports_started(void)
2501 struct rte_port *port;
2503 RTE_ETH_FOREACH_DEV(pi) {
2505 /* Check if there is a port which is not started */
2506 if ((port->port_status != RTE_PORT_STARTED) &&
2507 (port->slave_flag == 0))
2511 /* No port is not started */
2516 port_is_stopped(portid_t port_id)
2518 struct rte_port *port = &ports[port_id];
2520 if ((port->port_status != RTE_PORT_STOPPED) &&
2521 (port->slave_flag == 0))
2527 all_ports_stopped(void)
2531 RTE_ETH_FOREACH_DEV(pi) {
2532 if (!port_is_stopped(pi))
2540 port_is_started(portid_t port_id)
2542 if (port_id_is_invalid(port_id, ENABLED_WARN))
2545 if (ports[port_id].port_status != RTE_PORT_STARTED)
2551 /* Configure the Rx and Tx hairpin queues for the selected port. */
2553 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2556 struct rte_eth_hairpin_conf hairpin_conf = {
2561 struct rte_port *port = &ports[pi];
2562 uint16_t peer_rx_port = pi;
2563 uint16_t peer_tx_port = pi;
2564 uint32_t manual = 1;
2565 uint32_t tx_exp = hairpin_mode & 0x10;
2567 if (!(hairpin_mode & 0xf)) {
2571 } else if (hairpin_mode & 0x1) {
2572 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2573 RTE_ETH_DEV_NO_OWNER);
2574 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2575 peer_tx_port = rte_eth_find_next_owned_by(0,
2576 RTE_ETH_DEV_NO_OWNER);
2577 if (p_pi != RTE_MAX_ETHPORTS) {
2578 peer_rx_port = p_pi;
2582 /* Last port will be the peer RX port of the first. */
2583 RTE_ETH_FOREACH_DEV(next_pi)
2584 peer_rx_port = next_pi;
2587 } else if (hairpin_mode & 0x2) {
2589 peer_rx_port = p_pi;
2591 peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2592 RTE_ETH_DEV_NO_OWNER);
2593 if (peer_rx_port >= RTE_MAX_ETHPORTS)
2596 peer_tx_port = peer_rx_port;
2600 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2601 hairpin_conf.peers[0].port = peer_rx_port;
2602 hairpin_conf.peers[0].queue = i + nb_rxq;
2603 hairpin_conf.manual_bind = !!manual;
2604 hairpin_conf.tx_explicit = !!tx_exp;
2605 diag = rte_eth_tx_hairpin_queue_setup
2606 (pi, qi, nb_txd, &hairpin_conf);
2611 /* Fail to setup rx queue, return */
2612 if (port->port_status == RTE_PORT_HANDLING)
2613 port->port_status = RTE_PORT_STOPPED;
2616 "Port %d can not be set back to stopped\n", pi);
2617 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2619 /* try to reconfigure queues next time */
2620 port->need_reconfig_queues = 1;
2623 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2624 hairpin_conf.peers[0].port = peer_tx_port;
2625 hairpin_conf.peers[0].queue = i + nb_txq;
2626 hairpin_conf.manual_bind = !!manual;
2627 hairpin_conf.tx_explicit = !!tx_exp;
2628 diag = rte_eth_rx_hairpin_queue_setup
2629 (pi, qi, nb_rxd, &hairpin_conf);
2634 /* Fail to setup rx queue, return */
2635 if (port->port_status == RTE_PORT_HANDLING)
2636 port->port_status = RTE_PORT_STOPPED;
2639 "Port %d can not be set back to stopped\n", pi);
2640 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2642 /* try to reconfigure queues next time */
2643 port->need_reconfig_queues = 1;
2649 /* Configure the Rx with optional split. */
2651 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2652 uint16_t nb_rx_desc, unsigned int socket_id,
2653 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2655 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2656 unsigned int i, mp_n;
2659 if (rx_pkt_nb_segs <= 1 ||
2660 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2661 rx_conf->rx_seg = NULL;
2662 rx_conf->rx_nseg = 0;
2663 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2664 nb_rx_desc, socket_id,
2668 for (i = 0; i < rx_pkt_nb_segs; i++) {
2669 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2670 struct rte_mempool *mpx;
2672 * Use last valid pool for the segments with number
2673 * exceeding the pool index.
2675 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2676 mpx = mbuf_pool_find(socket_id, mp_n);
2677 /* Handle zero as mbuf data buffer size. */
2678 rx_seg->length = rx_pkt_seg_lengths[i] ?
2679 rx_pkt_seg_lengths[i] :
2680 mbuf_data_size[mp_n];
2681 rx_seg->offset = i < rx_pkt_nb_offs ?
2682 rx_pkt_seg_offsets[i] : 0;
2683 rx_seg->mp = mpx ? mpx : mp;
2685 rx_conf->rx_nseg = rx_pkt_nb_segs;
2686 rx_conf->rx_seg = rx_useg;
2687 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2688 socket_id, rx_conf, NULL);
2689 rx_conf->rx_seg = NULL;
2690 rx_conf->rx_nseg = 0;
2692 ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ?
2693 RTE_ETH_QUEUE_STATE_STOPPED :
2694 RTE_ETH_QUEUE_STATE_STARTED;
2699 alloc_xstats_display_info(portid_t pi)
2701 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2702 uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2703 uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2705 if (xstats_display_num == 0)
2708 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2709 if (*ids_supp == NULL)
2712 *prev_values = calloc(xstats_display_num,
2713 sizeof(**prev_values));
2714 if (*prev_values == NULL)
2715 goto fail_prev_values;
2717 *curr_values = calloc(xstats_display_num,
2718 sizeof(**curr_values));
2719 if (*curr_values == NULL)
2720 goto fail_curr_values;
2722 ports[pi].xstats_info.allocated = true;
2735 free_xstats_display_info(portid_t pi)
2737 if (!ports[pi].xstats_info.allocated)
2739 free(ports[pi].xstats_info.ids_supp);
2740 free(ports[pi].xstats_info.prev_values);
2741 free(ports[pi].xstats_info.curr_values);
2742 ports[pi].xstats_info.allocated = false;
2745 /** Fill helper structures for specified port to show extended statistics. */
2747 fill_xstats_display_info_for_port(portid_t pi)
2749 unsigned int stat, stat_supp;
2750 const char *xstat_name;
2751 struct rte_port *port;
2755 if (xstats_display_num == 0)
2758 if (pi == (portid_t)RTE_PORT_ALL) {
2759 fill_xstats_display_info();
2764 if (port->port_status != RTE_PORT_STARTED)
2767 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2768 rte_exit(EXIT_FAILURE,
2769 "Failed to allocate xstats display memory\n");
2771 ids_supp = port->xstats_info.ids_supp;
2772 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2773 xstat_name = xstats_display[stat].name;
2774 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2775 ids_supp + stat_supp);
2777 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2778 xstat_name, pi, stat);
2784 port->xstats_info.ids_supp_sz = stat_supp;
2787 /** Fill helper structures for all ports to show extended statistics. */
2789 fill_xstats_display_info(void)
2793 if (xstats_display_num == 0)
2796 RTE_ETH_FOREACH_DEV(pi)
2797 fill_xstats_display_info_for_port(pi);
2801 start_port(portid_t pid)
2803 int diag, need_check_link_status = -1;
2805 portid_t p_pi = RTE_MAX_ETHPORTS;
2806 portid_t pl[RTE_MAX_ETHPORTS];
2807 portid_t peer_pl[RTE_MAX_ETHPORTS];
2808 uint16_t cnt_pi = 0;
2809 uint16_t cfg_pi = 0;
2812 struct rte_port *port;
2813 struct rte_eth_hairpin_cap cap;
2815 if (port_id_is_invalid(pid, ENABLED_WARN))
2818 RTE_ETH_FOREACH_DEV(pi) {
2819 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2822 if (port_is_bonding_slave(pi)) {
2824 "Please remove port %d from bonded device.\n",
2829 need_check_link_status = 0;
2831 if (port->port_status == RTE_PORT_STOPPED)
2832 port->port_status = RTE_PORT_HANDLING;
2834 fprintf(stderr, "Port %d is now not stopped\n", pi);
2838 if (port->need_reconfig > 0) {
2839 struct rte_eth_conf dev_conf;
2842 port->need_reconfig = 0;
2844 if (flow_isolate_all) {
2845 int ret = port_flow_isolate(pi, 1);
2848 "Failed to apply isolated mode on port %d\n",
2853 configure_rxtx_dump_callbacks(0);
2854 printf("Configuring Port %d (socket %u)\n", pi,
2856 if (nb_hairpinq > 0 &&
2857 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2859 "Port %d doesn't support hairpin queues\n",
2864 /* configure port */
2865 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2866 nb_txq + nb_hairpinq,
2869 if (port->port_status == RTE_PORT_HANDLING)
2870 port->port_status = RTE_PORT_STOPPED;
2873 "Port %d can not be set back to stopped\n",
2875 fprintf(stderr, "Fail to configure port %d\n",
2877 /* try to reconfigure port next time */
2878 port->need_reconfig = 1;
2881 /* get device configuration*/
2883 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2885 "port %d can not get device configuration\n",
2889 /* Apply Rx offloads configuration */
2890 if (dev_conf.rxmode.offloads !=
2891 port->dev_conf.rxmode.offloads) {
2892 port->dev_conf.rxmode.offloads |=
2893 dev_conf.rxmode.offloads;
2895 k < port->dev_info.max_rx_queues;
2897 port->rxq[k].conf.offloads |=
2898 dev_conf.rxmode.offloads;
2900 /* Apply Tx offloads configuration */
2901 if (dev_conf.txmode.offloads !=
2902 port->dev_conf.txmode.offloads) {
2903 port->dev_conf.txmode.offloads |=
2904 dev_conf.txmode.offloads;
2906 k < port->dev_info.max_tx_queues;
2908 port->txq[k].conf.offloads |=
2909 dev_conf.txmode.offloads;
2912 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2913 port->need_reconfig_queues = 0;
2914 /* setup tx queues */
2915 for (qi = 0; qi < nb_txq; qi++) {
2916 struct rte_eth_txconf *conf =
2917 &port->txq[qi].conf;
2919 if ((numa_support) &&
2920 (txring_numa[pi] != NUMA_NO_CONFIG))
2921 diag = rte_eth_tx_queue_setup(pi, qi,
2922 port->nb_tx_desc[qi],
2924 &(port->txq[qi].conf));
2926 diag = rte_eth_tx_queue_setup(pi, qi,
2927 port->nb_tx_desc[qi],
2929 &(port->txq[qi].conf));
2932 port->txq[qi].state =
2933 conf->tx_deferred_start ?
2934 RTE_ETH_QUEUE_STATE_STOPPED :
2935 RTE_ETH_QUEUE_STATE_STARTED;
2939 /* Fail to setup tx queue, return */
2940 if (port->port_status == RTE_PORT_HANDLING)
2941 port->port_status = RTE_PORT_STOPPED;
2944 "Port %d can not be set back to stopped\n",
2947 "Fail to configure port %d tx queues\n",
2949 /* try to reconfigure queues next time */
2950 port->need_reconfig_queues = 1;
2953 for (qi = 0; qi < nb_rxq; qi++) {
2954 /* setup rx queues */
2955 if ((numa_support) &&
2956 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2957 struct rte_mempool * mp =
2959 (rxring_numa[pi], 0);
2962 "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2967 diag = rx_queue_setup(pi, qi,
2968 port->nb_rx_desc[qi],
2970 &(port->rxq[qi].conf),
2973 struct rte_mempool *mp =
2975 (port->socket_id, 0);
2978 "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2982 diag = rx_queue_setup(pi, qi,
2983 port->nb_rx_desc[qi],
2985 &(port->rxq[qi].conf),
2991 /* Fail to setup rx queue, return */
2992 if (port->port_status == RTE_PORT_HANDLING)
2993 port->port_status = RTE_PORT_STOPPED;
2996 "Port %d can not be set back to stopped\n",
2999 "Fail to configure port %d rx queues\n",
3001 /* try to reconfigure queues next time */
3002 port->need_reconfig_queues = 1;
3005 /* setup hairpin queues */
3006 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
3009 configure_rxtx_dump_callbacks(verbose_level);
3011 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
3015 "Port %d: Failed to disable Ptype parsing\n",
3023 diag = eth_dev_start_mp(pi);
3025 fprintf(stderr, "Fail to start port %d: %s\n",
3026 pi, rte_strerror(-diag));
3028 /* Fail to setup rx queue, return */
3029 if (port->port_status == RTE_PORT_HANDLING)
3030 port->port_status = RTE_PORT_STOPPED;
3033 "Port %d can not be set back to stopped\n",
3038 if (port->port_status == RTE_PORT_HANDLING)
3039 port->port_status = RTE_PORT_STARTED;
3041 fprintf(stderr, "Port %d can not be set into started\n",
3044 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
3045 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
3046 RTE_ETHER_ADDR_BYTES(&port->eth_addr));
3048 /* at least one port started, need checking link status */
3049 need_check_link_status = 1;
3054 if (need_check_link_status == 1 && !no_link_check)
3055 check_all_ports_link_status(RTE_PORT_ALL);
3056 else if (need_check_link_status == 0)
3057 fprintf(stderr, "Please stop the ports first\n");
3059 if (hairpin_mode & 0xf) {
3063 /* bind all started hairpin ports */
3064 for (i = 0; i < cfg_pi; i++) {
3066 /* bind current Tx to all peer Rx */
3067 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3068 RTE_MAX_ETHPORTS, 1);
3071 for (j = 0; j < peer_pi; j++) {
3072 if (!port_is_started(peer_pl[j]))
3074 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
3077 "Error during binding hairpin Tx port %u to %u: %s\n",
3079 rte_strerror(-diag));
3083 /* bind all peer Tx to current Rx */
3084 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3085 RTE_MAX_ETHPORTS, 0);
3088 for (j = 0; j < peer_pi; j++) {
3089 if (!port_is_started(peer_pl[j]))
3091 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
3094 "Error during binding hairpin Tx port %u to %u: %s\n",
3096 rte_strerror(-diag));
3103 fill_xstats_display_info_for_port(pid);
3110 stop_port(portid_t pid)
3113 struct rte_port *port;
3114 int need_check_link_status = 0;
3115 portid_t peer_pl[RTE_MAX_ETHPORTS];
3118 if (port_id_is_invalid(pid, ENABLED_WARN))
3121 printf("Stopping ports...\n");
3123 RTE_ETH_FOREACH_DEV(pi) {
3124 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3127 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3129 "Please remove port %d from forwarding configuration.\n",
3134 if (port_is_bonding_slave(pi)) {
3136 "Please remove port %d from bonded device.\n",
3142 if (port->port_status == RTE_PORT_STARTED)
3143 port->port_status = RTE_PORT_HANDLING;
3147 if (hairpin_mode & 0xf) {
3150 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3151 /* unbind all peer Tx from current Rx */
3152 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3153 RTE_MAX_ETHPORTS, 0);
3156 for (j = 0; j < peer_pi; j++) {
3157 if (!port_is_started(peer_pl[j]))
3159 rte_eth_hairpin_unbind(peer_pl[j], pi);
3163 if (port->flow_list)
3164 port_flow_flush(pi);
3166 if (eth_dev_stop_mp(pi) != 0)
3167 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3170 if (port->port_status == RTE_PORT_HANDLING)
3171 port->port_status = RTE_PORT_STOPPED;
3173 fprintf(stderr, "Port %d can not be set into stopped\n",
3175 need_check_link_status = 1;
3177 if (need_check_link_status && !no_link_check)
3178 check_all_ports_link_status(RTE_PORT_ALL);
3184 remove_invalid_ports_in(portid_t *array, portid_t *total)
3187 portid_t new_total = 0;
3189 for (i = 0; i < *total; i++)
3190 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3191 array[new_total] = array[i];
3198 remove_invalid_ports(void)
3200 remove_invalid_ports_in(ports_ids, &nb_ports);
3201 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3202 nb_cfg_ports = nb_fwd_ports;
3206 close_port(portid_t pid)
3209 struct rte_port *port;
3211 if (port_id_is_invalid(pid, ENABLED_WARN))
3214 printf("Closing ports...\n");
3216 RTE_ETH_FOREACH_DEV(pi) {
3217 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3220 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3222 "Please remove port %d from forwarding configuration.\n",
3227 if (port_is_bonding_slave(pi)) {
3229 "Please remove port %d from bonded device.\n",
3235 if (port->port_status == RTE_PORT_CLOSED) {
3236 fprintf(stderr, "Port %d is already closed\n", pi);
3240 if (is_proc_primary()) {
3241 mcast_addr_pool_destroy(pi);
3242 port_flow_flush(pi);
3243 port_flex_item_flush(pi);
3244 port_action_handle_flush(pi);
3245 rte_eth_dev_close(pi);
3248 free_xstats_display_info(pi);
3251 remove_invalid_ports();
3256 reset_port(portid_t pid)
3260 struct rte_port *port;
3262 if (port_id_is_invalid(pid, ENABLED_WARN))
3265 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3266 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3268 "Can not reset port(s), please stop port(s) first.\n");
3272 printf("Resetting ports...\n");
3274 RTE_ETH_FOREACH_DEV(pi) {
3275 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3278 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3280 "Please remove port %d from forwarding configuration.\n",
3285 if (port_is_bonding_slave(pi)) {
3287 "Please remove port %d from bonded device.\n",
3292 diag = rte_eth_dev_reset(pi);
3295 port->need_reconfig = 1;
3296 port->need_reconfig_queues = 1;
3298 fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3307 attach_port(char *identifier)
3310 struct rte_dev_iterator iterator;
3312 printf("Attaching a new port...\n");
3314 if (identifier == NULL) {
3315 fprintf(stderr, "Invalid parameters are specified\n");
3319 if (rte_dev_probe(identifier) < 0) {
3320 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3324 /* first attach mode: event */
3325 if (setup_on_probe_event) {
3326 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3327 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3328 if (ports[pi].port_status == RTE_PORT_HANDLING &&
3329 ports[pi].need_setup != 0)
3330 setup_attached_port(pi);
3334 /* second attach mode: iterator */
3335 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3336 /* setup ports matching the devargs used for probing */
3337 if (port_is_forwarding(pi))
3338 continue; /* port was already attached before */
3339 setup_attached_port(pi);
3344 setup_attached_port(portid_t pi)
3346 unsigned int socket_id;
3349 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3350 /* if socket_id is invalid, set to the first available socket. */
3351 if (check_socket_id(socket_id) < 0)
3352 socket_id = socket_ids[0];
3353 reconfig(pi, socket_id);
3354 ret = rte_eth_promiscuous_enable(pi);
3357 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3358 pi, rte_strerror(-ret));
3360 ports_ids[nb_ports++] = pi;
3361 fwd_ports_ids[nb_fwd_ports++] = pi;
3362 nb_cfg_ports = nb_fwd_ports;
3363 ports[pi].need_setup = 0;
3364 ports[pi].port_status = RTE_PORT_STOPPED;
3366 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3371 detach_device(struct rte_device *dev)
3376 fprintf(stderr, "Device already removed\n");
3380 printf("Removing a device...\n");
3382 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3383 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3384 if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3385 fprintf(stderr, "Port %u not stopped\n",
3389 port_flow_flush(sibling);
3393 if (rte_dev_remove(dev) < 0) {
3394 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3397 remove_invalid_ports();
3399 printf("Device is detached\n");
3400 printf("Now total ports is %d\n", nb_ports);
3406 detach_port_device(portid_t port_id)
3409 struct rte_eth_dev_info dev_info;
3411 if (port_id_is_invalid(port_id, ENABLED_WARN))
3414 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3415 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3416 fprintf(stderr, "Port not stopped\n");
3419 fprintf(stderr, "Port was not closed\n");
3422 ret = eth_dev_info_get_print_err(port_id, &dev_info);
3425 "Failed to get device info for port %d, not detaching\n",
3429 detach_device(dev_info.device);
3433 detach_devargs(char *identifier)
3435 struct rte_dev_iterator iterator;
3436 struct rte_devargs da;
3439 printf("Removing a device...\n");
3441 memset(&da, 0, sizeof(da));
3442 if (rte_devargs_parsef(&da, "%s", identifier)) {
3443 fprintf(stderr, "cannot parse identifier\n");
3447 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3448 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3449 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3450 fprintf(stderr, "Port %u not stopped\n",
3452 rte_eth_iterator_cleanup(&iterator);
3453 rte_devargs_reset(&da);
3456 port_flow_flush(port_id);
3460 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3461 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3462 da.name, da.bus->name);
3463 rte_devargs_reset(&da);
3467 remove_invalid_ports();
3469 printf("Device %s is detached\n", identifier);
3470 printf("Now total ports is %d\n", nb_ports);
3472 rte_devargs_reset(&da);
3483 stop_packet_forwarding();
3485 #ifndef RTE_EXEC_ENV_WINDOWS
3486 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3488 if (mp_alloc_type == MP_ALLOC_ANON)
3489 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3494 if (ports != NULL) {
3496 RTE_ETH_FOREACH_DEV(pt_id) {
3497 printf("\nStopping port %d...\n", pt_id);
3501 RTE_ETH_FOREACH_DEV(pt_id) {
3502 printf("\nShutting down port %d...\n", pt_id);
3509 ret = rte_dev_event_monitor_stop();
3512 "fail to stop device event monitor.");
3516 ret = rte_dev_event_callback_unregister(NULL,
3517 dev_event_callback, NULL);
3520 "fail to unregister device event callback.\n");
3524 ret = rte_dev_hotplug_handle_disable();
3527 "fail to disable hotplug handling.\n");
3531 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3533 mempool_free_mp(mempools[i]);
3535 free(xstats_display);
3537 printf("\nBye...\n");
3540 typedef void (*cmd_func_t)(void);
3541 struct pmd_test_command {
3542 const char *cmd_name;
3543 cmd_func_t cmd_func;
3546 /* Check the link status of all ports in up to 9s, and print them finally */
3548 check_all_ports_link_status(uint32_t port_mask)
3550 #define CHECK_INTERVAL 100 /* 100ms */
3551 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3553 uint8_t count, all_ports_up, print_flag = 0;
3554 struct rte_eth_link link;
3556 char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3558 printf("Checking link statuses...\n");
3560 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3562 RTE_ETH_FOREACH_DEV(portid) {
3563 if ((port_mask & (1 << portid)) == 0)
3565 memset(&link, 0, sizeof(link));
3566 ret = rte_eth_link_get_nowait(portid, &link);
3569 if (print_flag == 1)
3571 "Port %u link get failed: %s\n",
3572 portid, rte_strerror(-ret));
3575 /* print link status if flag set */
3576 if (print_flag == 1) {
3577 rte_eth_link_to_str(link_status,
3578 sizeof(link_status), &link);
3579 printf("Port %d %s\n", portid, link_status);
3582 /* clear all_ports_up flag if any link down */
3583 if (link.link_status == RTE_ETH_LINK_DOWN) {
3588 /* after finally printing all link status, get out */
3589 if (print_flag == 1)
3592 if (all_ports_up == 0) {
3594 rte_delay_ms(CHECK_INTERVAL);
3597 /* set the print_flag if all ports up or timeout */
3598 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3608 rmv_port_callback(void *arg)
3610 int need_to_start = 0;
3611 int org_no_link_check = no_link_check;
3612 portid_t port_id = (intptr_t)arg;
3613 struct rte_eth_dev_info dev_info;
3616 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3618 if (!test_done && port_is_forwarding(port_id)) {
3620 stop_packet_forwarding();
3624 no_link_check = org_no_link_check;
3626 ret = eth_dev_info_get_print_err(port_id, &dev_info);
3629 "Failed to get device info for port %d, not detaching\n",
3632 struct rte_device *device = dev_info.device;
3633 close_port(port_id);
3634 detach_device(device); /* might be already removed or have more ports */
3637 start_packet_forwarding(0);
3640 /* This function is used by the interrupt thread */
3642 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3645 RTE_SET_USED(param);
3646 RTE_SET_USED(ret_param);
3648 if (type >= RTE_ETH_EVENT_MAX) {
3650 "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3651 port_id, __func__, type);
3653 } else if (event_print_mask & (UINT32_C(1) << type)) {
3654 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3655 eth_event_desc[type]);
3660 case RTE_ETH_EVENT_NEW:
3661 ports[port_id].need_setup = 1;
3662 ports[port_id].port_status = RTE_PORT_HANDLING;
3664 case RTE_ETH_EVENT_INTR_RMV:
3665 if (port_id_is_invalid(port_id, DISABLED_WARN))
3667 if (rte_eal_alarm_set(100000,
3668 rmv_port_callback, (void *)(intptr_t)port_id))
3670 "Could not set up deferred device removal\n");
3672 case RTE_ETH_EVENT_DESTROY:
3673 ports[port_id].port_status = RTE_PORT_CLOSED;
3674 printf("Port %u is closed\n", port_id);
3676 case RTE_ETH_EVENT_RX_AVAIL_THRESH: {
3680 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */
3681 for (rxq_id = 0; ; rxq_id++) {
3682 ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id,
3686 printf("Received avail_thresh event, port: %u, rxq_id: %u\n",
3698 register_eth_event_callback(void)
3701 enum rte_eth_event_type event;
3703 for (event = RTE_ETH_EVENT_UNKNOWN;
3704 event < RTE_ETH_EVENT_MAX; event++) {
3705 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3710 TESTPMD_LOG(ERR, "Failed to register callback for "
3711 "%s event\n", eth_event_desc[event]);
3719 /* This function is used by the interrupt thread */
3721 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3722 __rte_unused void *arg)
3727 if (type >= RTE_DEV_EVENT_MAX) {
3728 fprintf(stderr, "%s called upon invalid event %d\n",
3734 case RTE_DEV_EVENT_REMOVE:
3735 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3737 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3739 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3744 * Because the user's callback is invoked in eal interrupt
3745 * callback, the interrupt callback need to be finished before
3746 * it can be unregistered when detaching device. So finish
3747 * callback soon and use a deferred removal to detach device
3748 * is need. It is a workaround, once the device detaching be
3749 * moved into the eal in the future, the deferred removal could
3752 if (rte_eal_alarm_set(100000,
3753 rmv_port_callback, (void *)(intptr_t)port_id))
3755 "Could not set up deferred device removal\n");
3757 case RTE_DEV_EVENT_ADD:
3758 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3760 /* TODO: After finish kernel driver binding,
3761 * begin to attach port.
3770 rxtx_port_config(portid_t pid)
3774 struct rte_port *port = &ports[pid];
3776 for (qid = 0; qid < nb_rxq; qid++) {
3777 offloads = port->rxq[qid].conf.offloads;
3778 port->rxq[qid].conf = port->dev_info.default_rxconf;
3780 if (rxq_share > 0 &&
3781 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3782 /* Non-zero share group to enable RxQ share. */
3783 port->rxq[qid].conf.share_group = pid / rxq_share + 1;
3784 port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */
3788 port->rxq[qid].conf.offloads = offloads;
3790 /* Check if any Rx parameters have been passed */
3791 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3792 port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh;
3794 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3795 port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh;
3797 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3798 port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh;
3800 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3801 port->rxq[qid].conf.rx_free_thresh = rx_free_thresh;
3803 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3804 port->rxq[qid].conf.rx_drop_en = rx_drop_en;
3806 port->nb_rx_desc[qid] = nb_rxd;
3809 for (qid = 0; qid < nb_txq; qid++) {
3810 offloads = port->txq[qid].conf.offloads;
3811 port->txq[qid].conf = port->dev_info.default_txconf;
3813 port->txq[qid].conf.offloads = offloads;
3815 /* Check if any Tx parameters have been passed */
3816 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3817 port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh;
3819 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3820 port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh;
3822 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3823 port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh;
3825 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3826 port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh;
3828 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3829 port->txq[qid].conf.tx_free_thresh = tx_free_thresh;
3831 port->nb_tx_desc[qid] = nb_txd;
3836 * Helper function to set MTU from frame size
3838 * port->dev_info should be set before calling this function.
3840 * return 0 on success, negative on error
3843 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3845 struct rte_port *port = &ports[portid];
3846 uint32_t eth_overhead;
3847 uint16_t mtu, new_mtu;
3849 eth_overhead = get_eth_overhead(&port->dev_info);
3851 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3852 printf("Failed to get MTU for port %u\n", portid);
3856 new_mtu = max_rx_pktlen - eth_overhead;
3861 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3863 "Failed to set MTU to %u for port %u\n",
3868 port->dev_conf.rxmode.mtu = new_mtu;
3874 init_port_config(void)
3877 struct rte_port *port;
3880 RTE_ETH_FOREACH_DEV(pid) {
3882 port->dev_conf.fdir_conf = fdir_conf;
3884 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3889 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3890 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3891 rss_hf & port->dev_info.flow_type_rss_offloads;
3893 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3894 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3897 if (port->dcb_flag == 0) {
3898 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3899 port->dev_conf.rxmode.mq_mode =
3900 (enum rte_eth_rx_mq_mode)
3901 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3903 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3904 port->dev_conf.rxmode.offloads &=
3905 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3908 i < port->dev_info.nb_rx_queues;
3910 port->rxq[i].conf.offloads &=
3911 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3915 rxtx_port_config(pid);
3917 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3921 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3922 rte_pmd_ixgbe_bypass_init(pid);
3925 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3926 port->dev_conf.intr_conf.lsc = 1;
3927 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3928 port->dev_conf.intr_conf.rmv = 1;
3932 void set_port_slave_flag(portid_t slave_pid)
3934 struct rte_port *port;
3936 port = &ports[slave_pid];
3937 port->slave_flag = 1;
3940 void clear_port_slave_flag(portid_t slave_pid)
3942 struct rte_port *port;
3944 port = &ports[slave_pid];
3945 port->slave_flag = 0;
3948 uint8_t port_is_bonding_slave(portid_t slave_pid)
3950 struct rte_port *port;
3951 struct rte_eth_dev_info dev_info;
3954 port = &ports[slave_pid];
3955 ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3958 "Failed to get device info for port id %d,"
3959 "cannot determine if the port is a bonded slave",
3963 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3968 const uint16_t vlan_tags[] = {
3969 0, 1, 2, 3, 4, 5, 6, 7,
3970 8, 9, 10, 11, 12, 13, 14, 15,
3971 16, 17, 18, 19, 20, 21, 22, 23,
3972 24, 25, 26, 27, 28, 29, 30, 31
3976 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3977 enum dcb_mode_enable dcb_mode,
3978 enum rte_eth_nb_tcs num_tcs,
3983 struct rte_eth_rss_conf rss_conf;
3986 * Builds up the correct configuration for dcb+vt based on the vlan tags array
3987 * given above, and the number of traffic classes available for use.
3989 if (dcb_mode == DCB_VT_ENABLED) {
3990 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3991 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3992 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3993 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3995 /* VMDQ+DCB RX and TX configurations */
3996 vmdq_rx_conf->enable_default_pool = 0;
3997 vmdq_rx_conf->default_pool = 0;
3998 vmdq_rx_conf->nb_queue_pools =
3999 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
4000 vmdq_tx_conf->nb_queue_pools =
4001 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
4003 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
4004 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
4005 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
4006 vmdq_rx_conf->pool_map[i].pools =
4007 1 << (i % vmdq_rx_conf->nb_queue_pools);
4009 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
4010 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
4011 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
4014 /* set DCB mode of RX and TX of multiple queues */
4015 eth_conf->rxmode.mq_mode =
4016 (enum rte_eth_rx_mq_mode)
4017 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
4018 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
4020 struct rte_eth_dcb_rx_conf *rx_conf =
4021 ð_conf->rx_adv_conf.dcb_rx_conf;
4022 struct rte_eth_dcb_tx_conf *tx_conf =
4023 ð_conf->tx_adv_conf.dcb_tx_conf;
4025 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
4027 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
4031 rx_conf->nb_tcs = num_tcs;
4032 tx_conf->nb_tcs = num_tcs;
4034 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
4035 rx_conf->dcb_tc[i] = i % num_tcs;
4036 tx_conf->dcb_tc[i] = i % num_tcs;
4039 eth_conf->rxmode.mq_mode =
4040 (enum rte_eth_rx_mq_mode)
4041 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
4042 eth_conf->rx_adv_conf.rss_conf = rss_conf;
4043 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
4047 eth_conf->dcb_capability_en =
4048 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
4050 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
4056 init_port_dcb_config(portid_t pid,
4057 enum dcb_mode_enable dcb_mode,
4058 enum rte_eth_nb_tcs num_tcs,
4061 struct rte_eth_conf port_conf;
4062 struct rte_port *rte_port;
4066 if (num_procs > 1) {
4067 printf("The multi-process feature doesn't support dcb.\n");
4070 rte_port = &ports[pid];
4072 /* retain the original device configuration. */
4073 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
4075 /*set configuration of DCB in vt mode and DCB in non-vt mode*/
4076 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
4079 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4080 /* remove RSS HASH offload for DCB in vt mode */
4081 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
4082 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4083 for (i = 0; i < nb_rxq; i++)
4084 rte_port->rxq[i].conf.offloads &=
4085 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4088 /* re-configure the device . */
4089 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
4093 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
4097 /* If dev_info.vmdq_pool_base is greater than 0,
4098 * the queue id of vmdq pools is started after pf queues.
4100 if (dcb_mode == DCB_VT_ENABLED &&
4101 rte_port->dev_info.vmdq_pool_base > 0) {
4103 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
4108 /* Assume the ports in testpmd have the same dcb capability
4109 * and has the same number of rxq and txq in dcb mode
4111 if (dcb_mode == DCB_VT_ENABLED) {
4112 if (rte_port->dev_info.max_vfs > 0) {
4113 nb_rxq = rte_port->dev_info.nb_rx_queues;
4114 nb_txq = rte_port->dev_info.nb_tx_queues;
4116 nb_rxq = rte_port->dev_info.max_rx_queues;
4117 nb_txq = rte_port->dev_info.max_tx_queues;
4120 /*if vt is disabled, use all pf queues */
4121 if (rte_port->dev_info.vmdq_pool_base == 0) {
4122 nb_rxq = rte_port->dev_info.max_rx_queues;
4123 nb_txq = rte_port->dev_info.max_tx_queues;
4125 nb_rxq = (queueid_t)num_tcs;
4126 nb_txq = (queueid_t)num_tcs;
4130 rx_free_thresh = 64;
4132 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4134 rxtx_port_config(pid);
4136 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4137 for (i = 0; i < RTE_DIM(vlan_tags); i++)
4138 rx_vft_set(pid, vlan_tags[i], 1);
4140 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4144 rte_port->dcb_flag = 1;
4146 /* Enter DCB configuration status */
4157 /* Configuration of Ethernet ports. */
4158 ports = rte_zmalloc("testpmd: ports",
4159 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4160 RTE_CACHE_LINE_SIZE);
4161 if (ports == NULL) {
4162 rte_exit(EXIT_FAILURE,
4163 "rte_zmalloc(%d struct rte_port) failed\n",
4166 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4167 ports[i].xstats_info.allocated = false;
4168 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4169 LIST_INIT(&ports[i].flow_tunnel_list);
4170 /* Initialize ports NUMA structures */
4171 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4172 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4173 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4187 const char clr[] = { 27, '[', '2', 'J', '\0' };
4188 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4190 /* Clear screen and move to top left */
4191 printf("%s%s", clr, top_left);
4193 printf("\nPort statistics ====================================");
4194 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4195 nic_stats_display(fwd_ports_ids[i]);
4201 signal_handler(int signum)
4203 if (signum == SIGINT || signum == SIGTERM) {
4204 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4206 #ifdef RTE_LIB_PDUMP
4207 /* uninitialize packet capture framework */
4210 #ifdef RTE_LIB_LATENCYSTATS
4211 if (latencystats_enabled != 0)
4212 rte_latencystats_uninit();
4215 /* Set flag to indicate the force termination. */
4217 /* exit with the expected status */
4218 #ifndef RTE_EXEC_ENV_WINDOWS
4219 signal(signum, SIG_DFL);
4220 kill(getpid(), signum);
4226 main(int argc, char** argv)
4233 signal(SIGINT, signal_handler);
4234 signal(SIGTERM, signal_handler);
4236 testpmd_logtype = rte_log_register("testpmd");
4237 if (testpmd_logtype < 0)
4238 rte_exit(EXIT_FAILURE, "Cannot register log type");
4239 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4241 diag = rte_eal_init(argc, argv);
4243 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4244 rte_strerror(rte_errno));
4246 ret = register_eth_event_callback();
4248 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4250 #ifdef RTE_LIB_PDUMP
4251 /* initialize packet capture framework */
4256 RTE_ETH_FOREACH_DEV(port_id) {
4257 ports_ids[count] = port_id;
4260 nb_ports = (portid_t) count;
4262 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4264 /* allocate port structures, and init them */
4267 set_def_fwd_config();
4269 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4270 "Check the core mask argument\n");
4272 /* Bitrate/latency stats disabled by default */
4273 #ifdef RTE_LIB_BITRATESTATS
4274 bitrate_enabled = 0;
4276 #ifdef RTE_LIB_LATENCYSTATS
4277 latencystats_enabled = 0;
4280 /* on FreeBSD, mlockall() is disabled by default */
4281 #ifdef RTE_EXEC_ENV_FREEBSD
4290 launch_args_parse(argc, argv);
4292 #ifndef RTE_EXEC_ENV_WINDOWS
4293 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4294 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4299 if (tx_first && interactive)
4300 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4301 "interactive mode.\n");
4303 if (tx_first && lsc_interrupt) {
4305 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4309 if (!nb_rxq && !nb_txq)
4311 "Warning: Either rx or tx queues should be non-zero\n");
4313 if (nb_rxq > 1 && nb_rxq > nb_txq)
4315 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4321 ret = rte_dev_hotplug_handle_enable();
4324 "fail to enable hotplug handling.");
4328 ret = rte_dev_event_monitor_start();
4331 "fail to start device event monitoring.");
4335 ret = rte_dev_event_callback_register(NULL,
4336 dev_event_callback, NULL);
4339 "fail to register device event callback\n");
4344 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4345 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4347 /* set all ports to promiscuous mode by default */
4348 RTE_ETH_FOREACH_DEV(port_id) {
4349 ret = rte_eth_promiscuous_enable(port_id);
4352 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4353 port_id, rte_strerror(-ret));
4356 #ifdef RTE_LIB_METRICS
4357 /* Init metrics library */
4358 rte_metrics_init(rte_socket_id());
4361 #ifdef RTE_LIB_LATENCYSTATS
4362 if (latencystats_enabled != 0) {
4363 int ret = rte_latencystats_init(1, NULL);
4366 "Warning: latencystats init() returned error %d\n",
4368 fprintf(stderr, "Latencystats running on lcore %d\n",
4369 latencystats_lcore_id);
4373 /* Setup bitrate stats */
4374 #ifdef RTE_LIB_BITRATESTATS
4375 if (bitrate_enabled != 0) {
4376 bitrate_data = rte_stats_bitrate_create();
4377 if (bitrate_data == NULL)
4378 rte_exit(EXIT_FAILURE,
4379 "Could not allocate bitrate data.\n");
4380 rte_stats_bitrate_reg(bitrate_data);
4383 #ifdef RTE_LIB_CMDLINE
4384 if (init_cmdline() != 0)
4385 rte_exit(EXIT_FAILURE,
4386 "Could not initialise cmdline context.\n");
4388 if (strlen(cmdline_filename) != 0)
4389 cmdline_read_from_file(cmdline_filename);
4391 if (interactive == 1) {
4393 printf("Start automatic packet forwarding\n");
4394 start_packet_forwarding(0);
4406 printf("No commandline core given, start packet forwarding\n");
4407 start_packet_forwarding(tx_first);
4408 if (stats_period != 0) {
4409 uint64_t prev_time = 0, cur_time, diff_time = 0;
4410 uint64_t timer_period;
4412 /* Convert to number of cycles */
4413 timer_period = stats_period * rte_get_timer_hz();
4415 while (f_quit == 0) {
4416 cur_time = rte_get_timer_cycles();
4417 diff_time += cur_time - prev_time;
4419 if (diff_time >= timer_period) {
4421 /* Reset the timer */
4424 /* Sleep to avoid unnecessary checks */
4425 prev_time = cur_time;
4426 rte_delay_us_sleep(US_PER_S);
4430 printf("Press enter to exit\n");
4431 rc = read(0, &c, 1);
4437 ret = rte_eal_cleanup();
4439 rte_exit(EXIT_FAILURE,
4440 "EAL cleanup failed: %s\n", strerror(-ret));
4442 return EXIT_SUCCESS;