1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
12 #ifndef RTE_EXEC_ENV_WINDOWS
15 #include <sys/types.h>
19 #include <sys/queue.h>
26 #include <rte_common.h>
27 #include <rte_errno.h>
28 #include <rte_byteorder.h>
30 #include <rte_debug.h>
31 #include <rte_cycles.h>
32 #include <rte_memory.h>
33 #include <rte_memcpy.h>
34 #include <rte_launch.h>
36 #include <rte_alarm.h>
37 #include <rte_per_lcore.h>
38 #include <rte_lcore.h>
39 #include <rte_atomic.h>
40 #include <rte_branch_prediction.h>
41 #include <rte_mempool.h>
42 #include <rte_malloc.h>
44 #include <rte_mbuf_pool_ops.h>
45 #include <rte_interrupts.h>
47 #include <rte_ether.h>
48 #include <rte_ethdev.h>
50 #include <rte_string_fns.h>
52 #include <rte_pmd_ixgbe.h>
55 #include <rte_pdump.h>
58 #include <rte_metrics.h>
59 #ifdef RTE_LIB_BITRATESTATS
60 #include <rte_bitrate.h>
62 #ifdef RTE_LIB_LATENCYSTATS
63 #include <rte_latencystats.h>
65 #ifdef RTE_EXEC_ENV_WINDOWS
72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
73 #define HUGE_FLAG (0x40000)
75 #define HUGE_FLAG MAP_HUGETLB
78 #ifndef MAP_HUGE_SHIFT
79 /* older kernels (or FreeBSD) will not have this define */
80 #define HUGE_SHIFT (26)
82 #define HUGE_SHIFT MAP_HUGE_SHIFT
85 #define EXTMEM_HEAP_NAME "extmem"
86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
88 uint16_t verbose_level = 0; /**< Silent by default. */
89 int testpmd_logtype; /**< Log type for testpmd logs */
91 /* use main core for command line ? */
92 uint8_t interactive = 0;
93 uint8_t auto_start = 0;
95 char cmdline_filename[PATH_MAX] = {0};
98 * NUMA support configuration.
99 * When set, the NUMA support attempts to dispatch the allocation of the
100 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
101 * probed ports among the CPU sockets 0 and 1.
102 * Otherwise, all memory is allocated from CPU socket 0.
104 uint8_t numa_support = 1; /**< numa enabled by default */
107 * In UMA mode,all memory is allocated from socket 0 if --socket-num is
110 uint8_t socket_num = UMA_NO_CONFIG;
113 * Select mempool allocation type:
114 * - native: use regular DPDK memory
115 * - anon: use regular DPDK memory to create mempool, but populate using
116 * anonymous memory (may not be IOVA-contiguous)
117 * - xmem: use externally allocated hugepage memory
119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
122 * Store specified sockets on which memory pool to be used by ports
125 uint8_t port_numa[RTE_MAX_ETHPORTS];
128 * Store specified sockets on which RX ring to be used by ports
131 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
134 * Store specified sockets on which TX ring to be used by ports
137 uint8_t txring_numa[RTE_MAX_ETHPORTS];
140 * Record the Ethernet address of peer target ports to which packets are
142 * Must be instantiated with the ethernet addresses of peer traffic generator
145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
146 portid_t nb_peer_eth_addrs = 0;
149 * Probed Target Environment.
151 struct rte_port *ports; /**< For all probed ethernet ports. */
152 portid_t nb_ports; /**< Number of probed ethernet ports. */
153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
154 lcoreid_t nb_lcores; /**< Number of probed logical cores. */
156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
159 * Test Forwarding Configuration.
160 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
161 * nb_fwd_ports <= nb_cfg_ports <= nb_ports
163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
165 portid_t nb_cfg_ports; /**< Number of configured ports. */
166 portid_t nb_fwd_ports; /**< Number of forwarding ports. */
168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */
171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
172 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */
175 * Forwarding engines.
177 struct fwd_engine * fwd_engines[] = {
187 &five_tuple_swap_fwd_engine,
188 #ifdef RTE_LIBRTE_IEEE1588
189 &ieee1588_fwd_engine,
194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
195 uint16_t mempool_flags;
197 struct fwd_config cur_fwd_config;
198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
199 uint32_t retry_enabled;
200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
205 DEFAULT_MBUF_DATA_SIZE
206 }; /**< Mbuf data space size. */
207 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if
208 * specified on command-line. */
209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
212 * In container, it cannot terminate the process which running with 'stats-period'
213 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
218 * Configuration of packet segments used to scatter received packets
219 * if some of split features is configured.
221 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
222 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */
223 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
224 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */
227 * Configuration of packet segments used by the "txonly" processing engine.
229 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
230 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
231 TXONLY_DEF_PACKET_LEN,
233 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
235 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
236 /**< Split policy for packets to TX. */
238 uint8_t txonly_multi_flow;
239 /**< Whether multiple flows are generated in TXONLY mode. */
241 uint32_t tx_pkt_times_inter;
242 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
244 uint32_t tx_pkt_times_intra;
245 /**< Timings for send scheduling in TXONLY mode, time between packets. */
247 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
248 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
249 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
251 /* current configuration is in DCB or not,0 means it is not in DCB mode */
252 uint8_t dcb_config = 0;
255 * Configurable number of RX/TX queues.
257 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
258 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
259 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
262 * Configurable number of RX/TX ring descriptors.
263 * Defaults are supplied by drivers via ethdev.
265 #define RTE_TEST_RX_DESC_DEFAULT 0
266 #define RTE_TEST_TX_DESC_DEFAULT 0
267 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
268 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
270 #define RTE_PMD_PARAM_UNSET -1
272 * Configurable values of RX and TX ring threshold registers.
275 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
276 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
277 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
279 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
280 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
281 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
284 * Configurable value of RX free threshold.
286 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
289 * Configurable value of RX drop enable.
291 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
294 * Configurable value of TX free threshold.
296 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
299 * Configurable value of TX RS bit threshold.
301 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
304 * Configurable value of buffered packets before sending.
306 uint16_t noisy_tx_sw_bufsz;
309 * Configurable value of packet buffer timeout.
311 uint16_t noisy_tx_sw_buf_flush_time;
314 * Configurable value for size of VNF internal memory area
315 * used for simulating noisy neighbour behaviour
317 uint64_t noisy_lkup_mem_sz;
320 * Configurable value of number of random writes done in
321 * VNF simulation memory area.
323 uint64_t noisy_lkup_num_writes;
326 * Configurable value of number of random reads done in
327 * VNF simulation memory area.
329 uint64_t noisy_lkup_num_reads;
332 * Configurable value of number of random reads/writes done in
333 * VNF simulation memory area.
335 uint64_t noisy_lkup_num_reads_writes;
338 * Receive Side Scaling (RSS) configuration.
340 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
343 * Port topology configuration
345 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
348 * Avoids to flush all the RX streams before starts forwarding.
350 uint8_t no_flush_rx = 0; /* flush by default */
353 * Flow API isolated mode.
355 uint8_t flow_isolate_all;
358 * Avoids to check link status when starting/stopping a port.
360 uint8_t no_link_check = 0; /* check by default */
363 * Don't automatically start all ports in interactive mode.
365 uint8_t no_device_start = 0;
368 * Enable link status change notification
370 uint8_t lsc_interrupt = 1; /* enabled by default */
373 * Enable device removal notification.
375 uint8_t rmv_interrupt = 1; /* enabled by default */
377 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
379 /* After attach, port setup is called on event or by iterator */
380 bool setup_on_probe_event = true;
382 /* Clear ptypes on port initialization. */
383 uint8_t clear_ptypes = true;
385 /* Hairpin ports configuration mode. */
386 uint16_t hairpin_mode;
388 /* Pretty printing of ethdev events */
389 static const char * const eth_event_desc[] = {
390 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
391 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
392 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
393 [RTE_ETH_EVENT_INTR_RESET] = "reset",
394 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
395 [RTE_ETH_EVENT_IPSEC] = "IPsec",
396 [RTE_ETH_EVENT_MACSEC] = "MACsec",
397 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
398 [RTE_ETH_EVENT_NEW] = "device probed",
399 [RTE_ETH_EVENT_DESTROY] = "device released",
400 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
401 [RTE_ETH_EVENT_MAX] = NULL,
405 * Display or mask ether events
406 * Default to all events except VF_MBOX
408 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
410 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
411 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
412 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
413 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
414 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
415 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
417 * Decide if all memory are locked for performance.
422 * NIC bypass mode configuration options.
425 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
426 /* The NIC bypass watchdog timeout. */
427 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
431 #ifdef RTE_LIB_LATENCYSTATS
434 * Set when latency stats is enabled in the commandline
436 uint8_t latencystats_enabled;
439 * Lcore ID to serive latency statistics.
441 lcoreid_t latencystats_lcore_id = -1;
446 * Ethernet device configuration.
448 struct rte_eth_rxmode rx_mode = {
449 /* Default maximum frame length.
450 * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
456 struct rte_eth_txmode tx_mode = {
457 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
460 struct rte_fdir_conf fdir_conf = {
461 .mode = RTE_FDIR_MODE_NONE,
462 .pballoc = RTE_FDIR_PBALLOC_64K,
463 .status = RTE_FDIR_REPORT_STATUS,
465 .vlan_tci_mask = 0xFFEF,
467 .src_ip = 0xFFFFFFFF,
468 .dst_ip = 0xFFFFFFFF,
471 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
472 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
474 .src_port_mask = 0xFFFF,
475 .dst_port_mask = 0xFFFF,
476 .mac_addr_byte_mask = 0xFF,
477 .tunnel_type_mask = 1,
478 .tunnel_id_mask = 0xFFFFFFFF,
483 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
486 * Display zero values by default for xstats
488 uint8_t xstats_hide_zero;
491 * Measure of CPU cycles disabled by default
493 uint8_t record_core_cycles;
496 * Display of RX and TX bursts disabled by default
498 uint8_t record_burst_stats;
500 unsigned int num_sockets = 0;
501 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
503 #ifdef RTE_LIB_BITRATESTATS
504 /* Bitrate statistics */
505 struct rte_stats_bitrates *bitrate_data;
506 lcoreid_t bitrate_lcore_id;
507 uint8_t bitrate_enabled;
510 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
511 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
514 * hexadecimal bitmask of RX mq mode can be enabled.
516 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
519 * Used to set forced link speed
521 uint32_t eth_link_speed;
523 /* Forward function declarations */
524 static void setup_attached_port(portid_t pi);
525 static void check_all_ports_link_status(uint32_t port_mask);
526 static int eth_event_callback(portid_t port_id,
527 enum rte_eth_event_type type,
528 void *param, void *ret_param);
529 static void dev_event_callback(const char *device_name,
530 enum rte_dev_event_type type,
534 * Check if all the ports are started.
535 * If yes, return positive value. If not, return zero.
537 static int all_ports_started(void);
539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
542 /* Holds the registered mbuf dynamic flags names. */
543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
546 * Helper function to check if socket is already discovered.
547 * If yes, return positive value. If not, return zero.
550 new_socket_id(unsigned int socket_id)
554 for (i = 0; i < num_sockets; i++) {
555 if (socket_ids[i] == socket_id)
562 * Setup default configuration.
565 set_default_fwd_lcores_config(void)
569 unsigned int sock_num;
572 for (i = 0; i < RTE_MAX_LCORE; i++) {
573 if (!rte_lcore_is_enabled(i))
575 sock_num = rte_lcore_to_socket_id(i);
576 if (new_socket_id(sock_num)) {
577 if (num_sockets >= RTE_MAX_NUMA_NODES) {
578 rte_exit(EXIT_FAILURE,
579 "Total sockets greater than %u\n",
582 socket_ids[num_sockets++] = sock_num;
584 if (i == rte_get_main_lcore())
586 fwd_lcores_cpuids[nb_lc++] = i;
588 nb_lcores = (lcoreid_t) nb_lc;
589 nb_cfg_lcores = nb_lcores;
594 set_def_peer_eth_addrs(void)
598 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
599 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
600 peer_eth_addrs[i].addr_bytes[5] = i;
605 set_default_fwd_ports_config(void)
610 RTE_ETH_FOREACH_DEV(pt_id) {
611 fwd_ports_ids[i++] = pt_id;
613 /* Update sockets info according to the attached device */
614 int socket_id = rte_eth_dev_socket_id(pt_id);
615 if (socket_id >= 0 && new_socket_id(socket_id)) {
616 if (num_sockets >= RTE_MAX_NUMA_NODES) {
617 rte_exit(EXIT_FAILURE,
618 "Total sockets greater than %u\n",
621 socket_ids[num_sockets++] = socket_id;
625 nb_cfg_ports = nb_ports;
626 nb_fwd_ports = nb_ports;
630 set_def_fwd_config(void)
632 set_default_fwd_lcores_config();
633 set_def_peer_eth_addrs();
634 set_default_fwd_ports_config();
637 #ifndef RTE_EXEC_ENV_WINDOWS
638 /* extremely pessimistic estimation of memory required to create a mempool */
640 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
642 unsigned int n_pages, mbuf_per_pg, leftover;
643 uint64_t total_mem, mbuf_mem, obj_sz;
645 /* there is no good way to predict how much space the mempool will
646 * occupy because it will allocate chunks on the fly, and some of those
647 * will come from default DPDK memory while some will come from our
648 * external memory, so just assume 128MB will be enough for everyone.
650 uint64_t hdr_mem = 128 << 20;
652 /* account for possible non-contiguousness */
653 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
655 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
659 mbuf_per_pg = pgsz / obj_sz;
660 leftover = (nb_mbufs % mbuf_per_pg) > 0;
661 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
663 mbuf_mem = n_pages * pgsz;
665 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
667 if (total_mem > SIZE_MAX) {
668 TESTPMD_LOG(ERR, "Memory size too big\n");
671 *out = (size_t)total_mem;
677 pagesz_flags(uint64_t page_sz)
679 /* as per mmap() manpage, all page sizes are log2 of page size
680 * shifted by MAP_HUGE_SHIFT
682 int log2 = rte_log2_u64(page_sz);
684 return (log2 << HUGE_SHIFT);
688 alloc_mem(size_t memsz, size_t pgsz, bool huge)
693 /* allocate anonymous hugepages */
694 flags = MAP_ANONYMOUS | MAP_PRIVATE;
696 flags |= HUGE_FLAG | pagesz_flags(pgsz);
698 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
699 if (addr == MAP_FAILED)
705 struct extmem_param {
709 rte_iova_t *iova_table;
710 unsigned int iova_table_len;
714 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
717 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
718 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */
719 unsigned int cur_page, n_pages, pgsz_idx;
720 size_t mem_sz, cur_pgsz;
721 rte_iova_t *iovas = NULL;
725 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
726 /* skip anything that is too big */
727 if (pgsizes[pgsz_idx] > SIZE_MAX)
730 cur_pgsz = pgsizes[pgsz_idx];
732 /* if we were told not to allocate hugepages, override */
734 cur_pgsz = sysconf(_SC_PAGESIZE);
736 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
738 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
742 /* allocate our memory */
743 addr = alloc_mem(mem_sz, cur_pgsz, huge);
745 /* if we couldn't allocate memory with a specified page size,
746 * that doesn't mean we can't do it with other page sizes, so
752 /* store IOVA addresses for every page in this memory area */
753 n_pages = mem_sz / cur_pgsz;
755 iovas = malloc(sizeof(*iovas) * n_pages);
758 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
761 /* lock memory if it's not huge pages */
765 /* populate IOVA addresses */
766 for (cur_page = 0; cur_page < n_pages; cur_page++) {
771 offset = cur_pgsz * cur_page;
772 cur = RTE_PTR_ADD(addr, offset);
774 /* touch the page before getting its IOVA */
775 *(volatile char *)cur = 0;
777 iova = rte_mem_virt2iova(cur);
779 iovas[cur_page] = iova;
784 /* if we couldn't allocate anything */
790 param->pgsz = cur_pgsz;
791 param->iova_table = iovas;
792 param->iova_table_len = n_pages;
799 munmap(addr, mem_sz);
805 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
807 struct extmem_param param;
810 memset(¶m, 0, sizeof(param));
812 /* check if our heap exists */
813 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
815 /* create our heap */
816 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
818 TESTPMD_LOG(ERR, "Cannot create heap\n");
823 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
825 TESTPMD_LOG(ERR, "Cannot create memory area\n");
829 /* we now have a valid memory area, so add it to heap */
830 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
831 param.addr, param.len, param.iova_table,
832 param.iova_table_len, param.pgsz);
834 /* when using VFIO, memory is automatically mapped for DMA by EAL */
836 /* not needed any more */
837 free(param.iova_table);
840 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
841 munmap(param.addr, param.len);
847 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
853 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
854 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
859 RTE_ETH_FOREACH_DEV(pid) {
860 struct rte_eth_dev_info dev_info;
862 ret = eth_dev_info_get_print_err(pid, &dev_info);
865 "unable to get device info for port %d on addr 0x%p,"
866 "mempool unmapping will not be performed\n",
871 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
874 "unable to DMA unmap addr 0x%p "
876 memhdr->addr, dev_info.device->name);
879 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
882 "unable to un-register addr 0x%p\n", memhdr->addr);
887 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
888 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
891 size_t page_size = sysconf(_SC_PAGESIZE);
894 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
898 "unable to register addr 0x%p\n", memhdr->addr);
901 RTE_ETH_FOREACH_DEV(pid) {
902 struct rte_eth_dev_info dev_info;
904 ret = eth_dev_info_get_print_err(pid, &dev_info);
907 "unable to get device info for port %d on addr 0x%p,"
908 "mempool mapping will not be performed\n",
912 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
915 "unable to DMA map addr 0x%p "
917 memhdr->addr, dev_info.device->name);
924 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
925 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
927 struct rte_pktmbuf_extmem *xmem;
928 unsigned int ext_num, zone_num, elt_num;
931 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
932 elt_num = EXTBUF_ZONE_SIZE / elt_size;
933 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
935 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
937 TESTPMD_LOG(ERR, "Cannot allocate memory for "
938 "external buffer descriptors\n");
942 for (ext_num = 0; ext_num < zone_num; ext_num++) {
943 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
944 const struct rte_memzone *mz;
945 char mz_name[RTE_MEMZONE_NAMESIZE];
948 ret = snprintf(mz_name, sizeof(mz_name),
949 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
950 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
951 errno = ENAMETOOLONG;
955 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
957 RTE_MEMZONE_IOVA_CONTIG |
959 RTE_MEMZONE_SIZE_HINT_ONLY,
963 * The caller exits on external buffer creation
964 * error, so there is no need to free memzones.
970 xseg->buf_ptr = mz->addr;
971 xseg->buf_iova = mz->iova;
972 xseg->buf_len = EXTBUF_ZONE_SIZE;
973 xseg->elt_size = elt_size;
975 if (ext_num == 0 && xmem != NULL) {
984 * Configuration initialisation done once at init time.
986 static struct rte_mempool *
987 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
988 unsigned int socket_id, uint16_t size_idx)
990 char pool_name[RTE_MEMPOOL_NAMESIZE];
991 struct rte_mempool *rte_mp = NULL;
992 #ifndef RTE_EXEC_ENV_WINDOWS
995 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
997 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1000 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1001 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1003 switch (mp_alloc_type) {
1004 case MP_ALLOC_NATIVE:
1006 /* wrapper to rte_mempool_create() */
1007 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1008 rte_mbuf_best_mempool_ops());
1009 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1010 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1013 #ifndef RTE_EXEC_ENV_WINDOWS
1016 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1017 mb_size, (unsigned int) mb_mempool_cache,
1018 sizeof(struct rte_pktmbuf_pool_private),
1019 socket_id, mempool_flags);
1023 if (rte_mempool_populate_anon(rte_mp) == 0) {
1024 rte_mempool_free(rte_mp);
1028 rte_pktmbuf_pool_init(rte_mp, NULL);
1029 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1030 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1034 case MP_ALLOC_XMEM_HUGE:
1037 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1039 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1040 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1043 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1044 if (heap_socket < 0)
1045 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1047 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1048 rte_mbuf_best_mempool_ops());
1049 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1050 mb_mempool_cache, 0, mbuf_seg_size,
1057 struct rte_pktmbuf_extmem *ext_mem;
1058 unsigned int ext_num;
1060 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1061 socket_id, pool_name, &ext_mem);
1063 rte_exit(EXIT_FAILURE,
1064 "Can't create pinned data buffers\n");
1066 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1067 rte_mbuf_best_mempool_ops());
1068 rte_mp = rte_pktmbuf_pool_create_extbuf
1069 (pool_name, nb_mbuf, mb_mempool_cache,
1070 0, mbuf_seg_size, socket_id,
1077 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1081 #ifndef RTE_EXEC_ENV_WINDOWS
1084 if (rte_mp == NULL) {
1085 rte_exit(EXIT_FAILURE,
1086 "Creation of mbuf pool for socket %u failed: %s\n",
1087 socket_id, rte_strerror(rte_errno));
1088 } else if (verbose_level > 0) {
1089 rte_mempool_dump(stdout, rte_mp);
1095 * Check given socket id is valid or not with NUMA mode,
1096 * if valid, return 0, else return -1
1099 check_socket_id(const unsigned int socket_id)
1101 static int warning_once = 0;
1103 if (new_socket_id(socket_id)) {
1104 if (!warning_once && numa_support)
1105 printf("Warning: NUMA should be configured manually by"
1106 " using --port-numa-config and"
1107 " --ring-numa-config parameters along with"
1116 * Get the allowed maximum number of RX queues.
1117 * *pid return the port id which has minimal value of
1118 * max_rx_queues in all ports.
1121 get_allowed_max_nb_rxq(portid_t *pid)
1123 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1124 bool max_rxq_valid = false;
1126 struct rte_eth_dev_info dev_info;
1128 RTE_ETH_FOREACH_DEV(pi) {
1129 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1132 max_rxq_valid = true;
1133 if (dev_info.max_rx_queues < allowed_max_rxq) {
1134 allowed_max_rxq = dev_info.max_rx_queues;
1138 return max_rxq_valid ? allowed_max_rxq : 0;
1142 * Check input rxq is valid or not.
1143 * If input rxq is not greater than any of maximum number
1144 * of RX queues of all ports, it is valid.
1145 * if valid, return 0, else return -1
1148 check_nb_rxq(queueid_t rxq)
1150 queueid_t allowed_max_rxq;
1153 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1154 if (rxq > allowed_max_rxq) {
1155 printf("Fail: input rxq (%u) can't be greater "
1156 "than max_rx_queues (%u) of port %u\n",
1166 * Get the allowed maximum number of TX queues.
1167 * *pid return the port id which has minimal value of
1168 * max_tx_queues in all ports.
1171 get_allowed_max_nb_txq(portid_t *pid)
1173 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1174 bool max_txq_valid = false;
1176 struct rte_eth_dev_info dev_info;
1178 RTE_ETH_FOREACH_DEV(pi) {
1179 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1182 max_txq_valid = true;
1183 if (dev_info.max_tx_queues < allowed_max_txq) {
1184 allowed_max_txq = dev_info.max_tx_queues;
1188 return max_txq_valid ? allowed_max_txq : 0;
1192 * Check input txq is valid or not.
1193 * If input txq is not greater than any of maximum number
1194 * of TX queues of all ports, it is valid.
1195 * if valid, return 0, else return -1
1198 check_nb_txq(queueid_t txq)
1200 queueid_t allowed_max_txq;
1203 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1204 if (txq > allowed_max_txq) {
1205 printf("Fail: input txq (%u) can't be greater "
1206 "than max_tx_queues (%u) of port %u\n",
1216 * Get the allowed maximum number of RXDs of every rx queue.
1217 * *pid return the port id which has minimal value of
1218 * max_rxd in all queues of all ports.
1221 get_allowed_max_nb_rxd(portid_t *pid)
1223 uint16_t allowed_max_rxd = UINT16_MAX;
1225 struct rte_eth_dev_info dev_info;
1227 RTE_ETH_FOREACH_DEV(pi) {
1228 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1231 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1232 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1236 return allowed_max_rxd;
1240 * Get the allowed minimal number of RXDs of every rx queue.
1241 * *pid return the port id which has minimal value of
1242 * min_rxd in all queues of all ports.
1245 get_allowed_min_nb_rxd(portid_t *pid)
1247 uint16_t allowed_min_rxd = 0;
1249 struct rte_eth_dev_info dev_info;
1251 RTE_ETH_FOREACH_DEV(pi) {
1252 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1255 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1256 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1261 return allowed_min_rxd;
1265 * Check input rxd is valid or not.
1266 * If input rxd is not greater than any of maximum number
1267 * of RXDs of every Rx queues and is not less than any of
1268 * minimal number of RXDs of every Rx queues, it is valid.
1269 * if valid, return 0, else return -1
1272 check_nb_rxd(queueid_t rxd)
1274 uint16_t allowed_max_rxd;
1275 uint16_t allowed_min_rxd;
1278 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1279 if (rxd > allowed_max_rxd) {
1280 printf("Fail: input rxd (%u) can't be greater "
1281 "than max_rxds (%u) of port %u\n",
1288 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1289 if (rxd < allowed_min_rxd) {
1290 printf("Fail: input rxd (%u) can't be less "
1291 "than min_rxds (%u) of port %u\n",
1302 * Get the allowed maximum number of TXDs of every rx queues.
1303 * *pid return the port id which has minimal value of
1304 * max_txd in every tx queue.
1307 get_allowed_max_nb_txd(portid_t *pid)
1309 uint16_t allowed_max_txd = UINT16_MAX;
1311 struct rte_eth_dev_info dev_info;
1313 RTE_ETH_FOREACH_DEV(pi) {
1314 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1317 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1318 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1322 return allowed_max_txd;
1326 * Get the allowed maximum number of TXDs of every tx queues.
1327 * *pid return the port id which has minimal value of
1328 * min_txd in every tx queue.
1331 get_allowed_min_nb_txd(portid_t *pid)
1333 uint16_t allowed_min_txd = 0;
1335 struct rte_eth_dev_info dev_info;
1337 RTE_ETH_FOREACH_DEV(pi) {
1338 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1341 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1342 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1347 return allowed_min_txd;
1351 * Check input txd is valid or not.
1352 * If input txd is not greater than any of maximum number
1353 * of TXDs of every Rx queues, it is valid.
1354 * if valid, return 0, else return -1
1357 check_nb_txd(queueid_t txd)
1359 uint16_t allowed_max_txd;
1360 uint16_t allowed_min_txd;
1363 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1364 if (txd > allowed_max_txd) {
1365 printf("Fail: input txd (%u) can't be greater "
1366 "than max_txds (%u) of port %u\n",
1373 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1374 if (txd < allowed_min_txd) {
1375 printf("Fail: input txd (%u) can't be less "
1376 "than min_txds (%u) of port %u\n",
1387 * Get the allowed maximum number of hairpin queues.
1388 * *pid return the port id which has minimal value of
1389 * max_hairpin_queues in all ports.
1392 get_allowed_max_nb_hairpinq(portid_t *pid)
1394 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1396 struct rte_eth_hairpin_cap cap;
1398 RTE_ETH_FOREACH_DEV(pi) {
1399 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1403 if (cap.max_nb_queues < allowed_max_hairpinq) {
1404 allowed_max_hairpinq = cap.max_nb_queues;
1408 return allowed_max_hairpinq;
1412 * Check input hairpin is valid or not.
1413 * If input hairpin is not greater than any of maximum number
1414 * of hairpin queues of all ports, it is valid.
1415 * if valid, return 0, else return -1
1418 check_nb_hairpinq(queueid_t hairpinq)
1420 queueid_t allowed_max_hairpinq;
1423 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1424 if (hairpinq > allowed_max_hairpinq) {
1425 printf("Fail: input hairpin (%u) can't be greater "
1426 "than max_hairpin_queues (%u) of port %u\n",
1427 hairpinq, allowed_max_hairpinq, pid);
1434 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1436 struct rte_port *port = &ports[pid];
1441 port->dev_conf.txmode = tx_mode;
1442 port->dev_conf.rxmode = rx_mode;
1444 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1446 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1448 ret = update_jumbo_frame_offload(pid);
1450 printf("Updating jumbo frame offload failed for port %u\n",
1453 if (!(port->dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1454 port->dev_conf.txmode.offloads &=
1455 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1457 /* Apply Rx offloads configuration */
1458 for (i = 0; i < port->dev_info.max_rx_queues; i++)
1459 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1460 /* Apply Tx offloads configuration */
1461 for (i = 0; i < port->dev_info.max_tx_queues; i++)
1462 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1465 port->dev_conf.link_speeds = eth_link_speed;
1467 /* set flag to initialize port/queue */
1468 port->need_reconfig = 1;
1469 port->need_reconfig_queues = 1;
1470 port->socket_id = socket_id;
1471 port->tx_metadata = 0;
1474 * Check for maximum number of segments per MTU.
1475 * Accordingly update the mbuf data size.
1477 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1478 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1479 data_size = rx_mode.max_rx_pkt_len /
1480 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1482 if ((data_size + RTE_PKTMBUF_HEADROOM) > mbuf_data_size[0]) {
1483 mbuf_data_size[0] = data_size + RTE_PKTMBUF_HEADROOM;
1484 TESTPMD_LOG(WARNING,
1485 "Configured mbuf size of the first segment %hu\n",
1495 struct rte_mempool *mbp;
1496 unsigned int nb_mbuf_per_pool;
1498 struct rte_gro_param gro_param;
1501 /* Configuration of logical cores. */
1502 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1503 sizeof(struct fwd_lcore *) * nb_lcores,
1504 RTE_CACHE_LINE_SIZE);
1505 if (fwd_lcores == NULL) {
1506 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1507 "failed\n", nb_lcores);
1509 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1510 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1511 sizeof(struct fwd_lcore),
1512 RTE_CACHE_LINE_SIZE);
1513 if (fwd_lcores[lc_id] == NULL) {
1514 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1517 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1520 RTE_ETH_FOREACH_DEV(pid) {
1524 socket_id = port_numa[pid];
1525 if (port_numa[pid] == NUMA_NO_CONFIG) {
1526 socket_id = rte_eth_dev_socket_id(pid);
1529 * if socket_id is invalid,
1530 * set to the first available socket.
1532 if (check_socket_id(socket_id) < 0)
1533 socket_id = socket_ids[0];
1536 socket_id = (socket_num == UMA_NO_CONFIG) ?
1539 /* Apply default TxRx configuration for all ports */
1540 init_config_port_offloads(pid, socket_id);
1543 * Create pools of mbuf.
1544 * If NUMA support is disabled, create a single pool of mbuf in
1545 * socket 0 memory by default.
1546 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1548 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1549 * nb_txd can be configured at run time.
1551 if (param_total_num_mbufs)
1552 nb_mbuf_per_pool = param_total_num_mbufs;
1554 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1555 (nb_lcores * mb_mempool_cache) +
1556 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1557 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1563 for (i = 0; i < num_sockets; i++)
1564 for (j = 0; j < mbuf_data_size_n; j++)
1565 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1566 mbuf_pool_create(mbuf_data_size[j],
1572 for (i = 0; i < mbuf_data_size_n; i++)
1573 mempools[i] = mbuf_pool_create
1576 socket_num == UMA_NO_CONFIG ?
1582 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1583 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1585 * Records which Mbuf pool to use by each logical core, if needed.
1587 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1588 mbp = mbuf_pool_find(
1589 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1592 mbp = mbuf_pool_find(0, 0);
1593 fwd_lcores[lc_id]->mbp = mbp;
1594 /* initialize GSO context */
1595 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1596 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1597 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1598 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1600 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1605 /* create a gro context for each lcore */
1606 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1607 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1608 gro_param.max_item_per_flow = MAX_PKT_BURST;
1609 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1610 gro_param.socket_id = rte_lcore_to_socket_id(
1611 fwd_lcores_cpuids[lc_id]);
1612 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1613 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1614 rte_exit(EXIT_FAILURE,
1615 "rte_gro_ctx_create() failed\n");
1622 reconfig(portid_t new_port_id, unsigned socket_id)
1624 /* Reconfiguration of Ethernet ports. */
1625 init_config_port_offloads(new_port_id, socket_id);
1631 init_fwd_streams(void)
1634 struct rte_port *port;
1635 streamid_t sm_id, nb_fwd_streams_new;
1638 /* set socket id according to numa or not */
1639 RTE_ETH_FOREACH_DEV(pid) {
1641 if (nb_rxq > port->dev_info.max_rx_queues) {
1642 printf("Fail: nb_rxq(%d) is greater than "
1643 "max_rx_queues(%d)\n", nb_rxq,
1644 port->dev_info.max_rx_queues);
1647 if (nb_txq > port->dev_info.max_tx_queues) {
1648 printf("Fail: nb_txq(%d) is greater than "
1649 "max_tx_queues(%d)\n", nb_txq,
1650 port->dev_info.max_tx_queues);
1654 if (port_numa[pid] != NUMA_NO_CONFIG)
1655 port->socket_id = port_numa[pid];
1657 port->socket_id = rte_eth_dev_socket_id(pid);
1660 * if socket_id is invalid,
1661 * set to the first available socket.
1663 if (check_socket_id(port->socket_id) < 0)
1664 port->socket_id = socket_ids[0];
1668 if (socket_num == UMA_NO_CONFIG)
1669 port->socket_id = 0;
1671 port->socket_id = socket_num;
1675 q = RTE_MAX(nb_rxq, nb_txq);
1677 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1680 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1681 if (nb_fwd_streams_new == nb_fwd_streams)
1684 if (fwd_streams != NULL) {
1685 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1686 if (fwd_streams[sm_id] == NULL)
1688 rte_free(fwd_streams[sm_id]);
1689 fwd_streams[sm_id] = NULL;
1691 rte_free(fwd_streams);
1696 nb_fwd_streams = nb_fwd_streams_new;
1697 if (nb_fwd_streams) {
1698 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1699 sizeof(struct fwd_stream *) * nb_fwd_streams,
1700 RTE_CACHE_LINE_SIZE);
1701 if (fwd_streams == NULL)
1702 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1703 " (struct fwd_stream *)) failed\n",
1706 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1707 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1708 " struct fwd_stream", sizeof(struct fwd_stream),
1709 RTE_CACHE_LINE_SIZE);
1710 if (fwd_streams[sm_id] == NULL)
1711 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1712 "(struct fwd_stream) failed\n");
1720 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1722 uint64_t total_burst, sburst;
1724 uint64_t burst_stats[4];
1725 uint16_t pktnb_stats[4];
1727 int burst_percent[4], sburstp;
1731 * First compute the total number of packet bursts and the
1732 * two highest numbers of bursts of the same number of packets.
1734 memset(&burst_stats, 0x0, sizeof(burst_stats));
1735 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1737 /* Show stats for 0 burst size always */
1738 total_burst = pbs->pkt_burst_spread[0];
1739 burst_stats[0] = pbs->pkt_burst_spread[0];
1742 /* Find the next 2 burst sizes with highest occurrences. */
1743 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1744 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1749 total_burst += nb_burst;
1751 if (nb_burst > burst_stats[1]) {
1752 burst_stats[2] = burst_stats[1];
1753 pktnb_stats[2] = pktnb_stats[1];
1754 burst_stats[1] = nb_burst;
1755 pktnb_stats[1] = nb_pkt;
1756 } else if (nb_burst > burst_stats[2]) {
1757 burst_stats[2] = nb_burst;
1758 pktnb_stats[2] = nb_pkt;
1761 if (total_burst == 0)
1764 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1765 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1767 printf("%d%% of other]\n", 100 - sburstp);
1771 sburst += burst_stats[i];
1772 if (sburst == total_burst) {
1773 printf("%d%% of %d pkts]\n",
1774 100 - sburstp, (int) pktnb_stats[i]);
1779 (double)burst_stats[i] / total_burst * 100;
1780 printf("%d%% of %d pkts + ",
1781 burst_percent[i], (int) pktnb_stats[i]);
1782 sburstp += burst_percent[i];
1787 fwd_stream_stats_display(streamid_t stream_id)
1789 struct fwd_stream *fs;
1790 static const char *fwd_top_stats_border = "-------";
1792 fs = fwd_streams[stream_id];
1793 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1794 (fs->fwd_dropped == 0))
1796 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1797 "TX Port=%2d/Queue=%2d %s\n",
1798 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1799 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1800 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1801 " TX-dropped: %-14"PRIu64,
1802 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1804 /* if checksum mode */
1805 if (cur_fwd_eng == &csum_fwd_engine) {
1806 printf(" RX- bad IP checksum: %-14"PRIu64
1807 " Rx- bad L4 checksum: %-14"PRIu64
1808 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1809 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1810 fs->rx_bad_outer_l4_csum);
1811 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1812 fs->rx_bad_outer_ip_csum);
1817 if (record_burst_stats) {
1818 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1819 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1824 fwd_stats_display(void)
1826 static const char *fwd_stats_border = "----------------------";
1827 static const char *acc_stats_border = "+++++++++++++++";
1829 struct fwd_stream *rx_stream;
1830 struct fwd_stream *tx_stream;
1831 uint64_t tx_dropped;
1832 uint64_t rx_bad_ip_csum;
1833 uint64_t rx_bad_l4_csum;
1834 uint64_t rx_bad_outer_l4_csum;
1835 uint64_t rx_bad_outer_ip_csum;
1836 } ports_stats[RTE_MAX_ETHPORTS];
1837 uint64_t total_rx_dropped = 0;
1838 uint64_t total_tx_dropped = 0;
1839 uint64_t total_rx_nombuf = 0;
1840 struct rte_eth_stats stats;
1841 uint64_t fwd_cycles = 0;
1842 uint64_t total_recv = 0;
1843 uint64_t total_xmit = 0;
1844 struct rte_port *port;
1849 memset(ports_stats, 0, sizeof(ports_stats));
1851 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1852 struct fwd_stream *fs = fwd_streams[sm_id];
1854 if (cur_fwd_config.nb_fwd_streams >
1855 cur_fwd_config.nb_fwd_ports) {
1856 fwd_stream_stats_display(sm_id);
1858 ports_stats[fs->tx_port].tx_stream = fs;
1859 ports_stats[fs->rx_port].rx_stream = fs;
1862 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1864 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1865 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1866 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1867 fs->rx_bad_outer_l4_csum;
1868 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1869 fs->rx_bad_outer_ip_csum;
1871 if (record_core_cycles)
1872 fwd_cycles += fs->core_cycles;
1874 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1875 pt_id = fwd_ports_ids[i];
1876 port = &ports[pt_id];
1878 rte_eth_stats_get(pt_id, &stats);
1879 stats.ipackets -= port->stats.ipackets;
1880 stats.opackets -= port->stats.opackets;
1881 stats.ibytes -= port->stats.ibytes;
1882 stats.obytes -= port->stats.obytes;
1883 stats.imissed -= port->stats.imissed;
1884 stats.oerrors -= port->stats.oerrors;
1885 stats.rx_nombuf -= port->stats.rx_nombuf;
1887 total_recv += stats.ipackets;
1888 total_xmit += stats.opackets;
1889 total_rx_dropped += stats.imissed;
1890 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1891 total_tx_dropped += stats.oerrors;
1892 total_rx_nombuf += stats.rx_nombuf;
1894 printf("\n %s Forward statistics for port %-2d %s\n",
1895 fwd_stats_border, pt_id, fwd_stats_border);
1897 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1898 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1899 stats.ipackets + stats.imissed);
1901 if (cur_fwd_eng == &csum_fwd_engine) {
1902 printf(" Bad-ipcsum: %-14"PRIu64
1903 " Bad-l4csum: %-14"PRIu64
1904 "Bad-outer-l4csum: %-14"PRIu64"\n",
1905 ports_stats[pt_id].rx_bad_ip_csum,
1906 ports_stats[pt_id].rx_bad_l4_csum,
1907 ports_stats[pt_id].rx_bad_outer_l4_csum);
1908 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n",
1909 ports_stats[pt_id].rx_bad_outer_ip_csum);
1911 if (stats.ierrors + stats.rx_nombuf > 0) {
1912 printf(" RX-error: %-"PRIu64"\n", stats.ierrors);
1913 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1916 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1917 "TX-total: %-"PRIu64"\n",
1918 stats.opackets, ports_stats[pt_id].tx_dropped,
1919 stats.opackets + ports_stats[pt_id].tx_dropped);
1921 if (record_burst_stats) {
1922 if (ports_stats[pt_id].rx_stream)
1923 pkt_burst_stats_display("RX",
1924 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1925 if (ports_stats[pt_id].tx_stream)
1926 pkt_burst_stats_display("TX",
1927 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1930 printf(" %s--------------------------------%s\n",
1931 fwd_stats_border, fwd_stats_border);
1934 printf("\n %s Accumulated forward statistics for all ports"
1936 acc_stats_border, acc_stats_border);
1937 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1939 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1941 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1942 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1943 if (total_rx_nombuf > 0)
1944 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1945 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
1947 acc_stats_border, acc_stats_border);
1948 if (record_core_cycles) {
1949 #define CYC_PER_MHZ 1E6
1950 if (total_recv > 0 || total_xmit > 0) {
1951 uint64_t total_pkts = 0;
1952 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1953 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1954 total_pkts = total_xmit;
1956 total_pkts = total_recv;
1958 printf("\n CPU cycles/packet=%.2F (total cycles="
1959 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1961 (double) fwd_cycles / total_pkts,
1962 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1963 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1969 fwd_stats_reset(void)
1975 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1976 pt_id = fwd_ports_ids[i];
1977 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1979 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1980 struct fwd_stream *fs = fwd_streams[sm_id];
1984 fs->fwd_dropped = 0;
1985 fs->rx_bad_ip_csum = 0;
1986 fs->rx_bad_l4_csum = 0;
1987 fs->rx_bad_outer_l4_csum = 0;
1988 fs->rx_bad_outer_ip_csum = 0;
1990 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1991 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1992 fs->core_cycles = 0;
1997 flush_fwd_rx_queues(void)
1999 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2006 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2007 uint64_t timer_period;
2009 /* convert to number of cycles */
2010 timer_period = rte_get_timer_hz(); /* 1 second timeout */
2012 for (j = 0; j < 2; j++) {
2013 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2014 for (rxq = 0; rxq < nb_rxq; rxq++) {
2015 port_id = fwd_ports_ids[rxp];
2017 * testpmd can stuck in the below do while loop
2018 * if rte_eth_rx_burst() always returns nonzero
2019 * packets. So timer is added to exit this loop
2020 * after 1sec timer expiry.
2022 prev_tsc = rte_rdtsc();
2024 nb_rx = rte_eth_rx_burst(port_id, rxq,
2025 pkts_burst, MAX_PKT_BURST);
2026 for (i = 0; i < nb_rx; i++)
2027 rte_pktmbuf_free(pkts_burst[i]);
2029 cur_tsc = rte_rdtsc();
2030 diff_tsc = cur_tsc - prev_tsc;
2031 timer_tsc += diff_tsc;
2032 } while ((nb_rx > 0) &&
2033 (timer_tsc < timer_period));
2037 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2042 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2044 struct fwd_stream **fsm;
2047 #ifdef RTE_LIB_BITRATESTATS
2048 uint64_t tics_per_1sec;
2049 uint64_t tics_datum;
2050 uint64_t tics_current;
2051 uint16_t i, cnt_ports;
2053 cnt_ports = nb_ports;
2054 tics_datum = rte_rdtsc();
2055 tics_per_1sec = rte_get_timer_hz();
2057 fsm = &fwd_streams[fc->stream_idx];
2058 nb_fs = fc->stream_nb;
2060 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2061 (*pkt_fwd)(fsm[sm_id]);
2062 #ifdef RTE_LIB_BITRATESTATS
2063 if (bitrate_enabled != 0 &&
2064 bitrate_lcore_id == rte_lcore_id()) {
2065 tics_current = rte_rdtsc();
2066 if (tics_current - tics_datum >= tics_per_1sec) {
2067 /* Periodic bitrate calculation */
2068 for (i = 0; i < cnt_ports; i++)
2069 rte_stats_bitrate_calc(bitrate_data,
2071 tics_datum = tics_current;
2075 #ifdef RTE_LIB_LATENCYSTATS
2076 if (latencystats_enabled != 0 &&
2077 latencystats_lcore_id == rte_lcore_id())
2078 rte_latencystats_update();
2081 } while (! fc->stopped);
2085 start_pkt_forward_on_core(void *fwd_arg)
2087 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2088 cur_fwd_config.fwd_eng->packet_fwd);
2093 * Run the TXONLY packet forwarding engine to send a single burst of packets.
2094 * Used to start communication flows in network loopback test configurations.
2097 run_one_txonly_burst_on_core(void *fwd_arg)
2099 struct fwd_lcore *fwd_lc;
2100 struct fwd_lcore tmp_lcore;
2102 fwd_lc = (struct fwd_lcore *) fwd_arg;
2103 tmp_lcore = *fwd_lc;
2104 tmp_lcore.stopped = 1;
2105 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2110 * Launch packet forwarding:
2111 * - Setup per-port forwarding context.
2112 * - launch logical cores with their forwarding configuration.
2115 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2117 port_fwd_begin_t port_fwd_begin;
2122 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2123 if (port_fwd_begin != NULL) {
2124 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2125 (*port_fwd_begin)(fwd_ports_ids[i]);
2127 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2128 lc_id = fwd_lcores_cpuids[i];
2129 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2130 fwd_lcores[i]->stopped = 0;
2131 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2132 fwd_lcores[i], lc_id);
2134 printf("launch lcore %u failed - diag=%d\n",
2141 * Launch packet forwarding configuration.
2144 start_packet_forwarding(int with_tx_first)
2146 port_fwd_begin_t port_fwd_begin;
2147 port_fwd_end_t port_fwd_end;
2150 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2151 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2153 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2154 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2156 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2157 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2158 (!nb_rxq || !nb_txq))
2159 rte_exit(EXIT_FAILURE,
2160 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2161 cur_fwd_eng->fwd_mode_name);
2163 if (all_ports_started() == 0) {
2164 printf("Not all ports were started\n");
2167 if (test_done == 0) {
2168 printf("Packet forwarding already started\n");
2176 flush_fwd_rx_queues();
2178 pkt_fwd_config_display(&cur_fwd_config);
2179 rxtx_config_display();
2182 if (with_tx_first) {
2183 port_fwd_begin = tx_only_engine.port_fwd_begin;
2184 if (port_fwd_begin != NULL) {
2185 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2186 (*port_fwd_begin)(fwd_ports_ids[i]);
2188 while (with_tx_first--) {
2189 launch_packet_forwarding(
2190 run_one_txonly_burst_on_core);
2191 rte_eal_mp_wait_lcore();
2193 port_fwd_end = tx_only_engine.port_fwd_end;
2194 if (port_fwd_end != NULL) {
2195 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2196 (*port_fwd_end)(fwd_ports_ids[i]);
2199 launch_packet_forwarding(start_pkt_forward_on_core);
2203 stop_packet_forwarding(void)
2205 port_fwd_end_t port_fwd_end;
2211 printf("Packet forwarding not started\n");
2214 printf("Telling cores to stop...");
2215 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2216 fwd_lcores[lc_id]->stopped = 1;
2217 printf("\nWaiting for lcores to finish...\n");
2218 rte_eal_mp_wait_lcore();
2219 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2220 if (port_fwd_end != NULL) {
2221 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2222 pt_id = fwd_ports_ids[i];
2223 (*port_fwd_end)(pt_id);
2227 fwd_stats_display();
2229 printf("\nDone.\n");
2234 dev_set_link_up(portid_t pid)
2236 if (rte_eth_dev_set_link_up(pid) < 0)
2237 printf("\nSet link up fail.\n");
2241 dev_set_link_down(portid_t pid)
2243 if (rte_eth_dev_set_link_down(pid) < 0)
2244 printf("\nSet link down fail.\n");
2248 all_ports_started(void)
2251 struct rte_port *port;
2253 RTE_ETH_FOREACH_DEV(pi) {
2255 /* Check if there is a port which is not started */
2256 if ((port->port_status != RTE_PORT_STARTED) &&
2257 (port->slave_flag == 0))
2261 /* No port is not started */
2266 port_is_stopped(portid_t port_id)
2268 struct rte_port *port = &ports[port_id];
2270 if ((port->port_status != RTE_PORT_STOPPED) &&
2271 (port->slave_flag == 0))
2277 all_ports_stopped(void)
2281 RTE_ETH_FOREACH_DEV(pi) {
2282 if (!port_is_stopped(pi))
2290 port_is_started(portid_t port_id)
2292 if (port_id_is_invalid(port_id, ENABLED_WARN))
2295 if (ports[port_id].port_status != RTE_PORT_STARTED)
2301 /* Configure the Rx and Tx hairpin queues for the selected port. */
2303 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2306 struct rte_eth_hairpin_conf hairpin_conf = {
2311 struct rte_port *port = &ports[pi];
2312 uint16_t peer_rx_port = pi;
2313 uint16_t peer_tx_port = pi;
2314 uint32_t manual = 1;
2315 uint32_t tx_exp = hairpin_mode & 0x10;
2317 if (!(hairpin_mode & 0xf)) {
2321 } else if (hairpin_mode & 0x1) {
2322 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2323 RTE_ETH_DEV_NO_OWNER);
2324 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2325 peer_tx_port = rte_eth_find_next_owned_by(0,
2326 RTE_ETH_DEV_NO_OWNER);
2327 if (p_pi != RTE_MAX_ETHPORTS) {
2328 peer_rx_port = p_pi;
2332 /* Last port will be the peer RX port of the first. */
2333 RTE_ETH_FOREACH_DEV(next_pi)
2334 peer_rx_port = next_pi;
2337 } else if (hairpin_mode & 0x2) {
2339 peer_rx_port = p_pi;
2341 peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2342 RTE_ETH_DEV_NO_OWNER);
2343 if (peer_rx_port >= RTE_MAX_ETHPORTS)
2346 peer_tx_port = peer_rx_port;
2350 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2351 hairpin_conf.peers[0].port = peer_rx_port;
2352 hairpin_conf.peers[0].queue = i + nb_rxq;
2353 hairpin_conf.manual_bind = !!manual;
2354 hairpin_conf.tx_explicit = !!tx_exp;
2355 diag = rte_eth_tx_hairpin_queue_setup
2356 (pi, qi, nb_txd, &hairpin_conf);
2361 /* Fail to setup rx queue, return */
2362 if (rte_atomic16_cmpset(&(port->port_status),
2364 RTE_PORT_STOPPED) == 0)
2365 printf("Port %d can not be set back "
2366 "to stopped\n", pi);
2367 printf("Fail to configure port %d hairpin "
2369 /* try to reconfigure queues next time */
2370 port->need_reconfig_queues = 1;
2373 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2374 hairpin_conf.peers[0].port = peer_tx_port;
2375 hairpin_conf.peers[0].queue = i + nb_txq;
2376 hairpin_conf.manual_bind = !!manual;
2377 hairpin_conf.tx_explicit = !!tx_exp;
2378 diag = rte_eth_rx_hairpin_queue_setup
2379 (pi, qi, nb_rxd, &hairpin_conf);
2384 /* Fail to setup rx queue, return */
2385 if (rte_atomic16_cmpset(&(port->port_status),
2387 RTE_PORT_STOPPED) == 0)
2388 printf("Port %d can not be set back "
2389 "to stopped\n", pi);
2390 printf("Fail to configure port %d hairpin "
2392 /* try to reconfigure queues next time */
2393 port->need_reconfig_queues = 1;
2399 /* Configure the Rx with optional split. */
2401 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2402 uint16_t nb_rx_desc, unsigned int socket_id,
2403 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2405 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2406 unsigned int i, mp_n;
2409 if (rx_pkt_nb_segs <= 1 ||
2410 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2411 rx_conf->rx_seg = NULL;
2412 rx_conf->rx_nseg = 0;
2413 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2414 nb_rx_desc, socket_id,
2418 for (i = 0; i < rx_pkt_nb_segs; i++) {
2419 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2420 struct rte_mempool *mpx;
2422 * Use last valid pool for the segments with number
2423 * exceeding the pool index.
2425 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2426 mpx = mbuf_pool_find(socket_id, mp_n);
2427 /* Handle zero as mbuf data buffer size. */
2428 rx_seg->length = rx_pkt_seg_lengths[i] ?
2429 rx_pkt_seg_lengths[i] :
2430 mbuf_data_size[mp_n];
2431 rx_seg->offset = i < rx_pkt_nb_offs ?
2432 rx_pkt_seg_offsets[i] : 0;
2433 rx_seg->mp = mpx ? mpx : mp;
2435 rx_conf->rx_nseg = rx_pkt_nb_segs;
2436 rx_conf->rx_seg = rx_useg;
2437 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2438 socket_id, rx_conf, NULL);
2439 rx_conf->rx_seg = NULL;
2440 rx_conf->rx_nseg = 0;
2445 start_port(portid_t pid)
2447 int diag, need_check_link_status = -1;
2449 portid_t p_pi = RTE_MAX_ETHPORTS;
2450 portid_t pl[RTE_MAX_ETHPORTS];
2451 portid_t peer_pl[RTE_MAX_ETHPORTS];
2452 uint16_t cnt_pi = 0;
2453 uint16_t cfg_pi = 0;
2456 struct rte_port *port;
2457 struct rte_eth_hairpin_cap cap;
2459 if (port_id_is_invalid(pid, ENABLED_WARN))
2462 RTE_ETH_FOREACH_DEV(pi) {
2463 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2466 need_check_link_status = 0;
2468 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2469 RTE_PORT_HANDLING) == 0) {
2470 printf("Port %d is now not stopped\n", pi);
2474 if (port->need_reconfig > 0) {
2475 port->need_reconfig = 0;
2477 if (flow_isolate_all) {
2478 int ret = port_flow_isolate(pi, 1);
2480 printf("Failed to apply isolated"
2481 " mode on port %d\n", pi);
2485 configure_rxtx_dump_callbacks(0);
2486 printf("Configuring Port %d (socket %u)\n", pi,
2488 if (nb_hairpinq > 0 &&
2489 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2490 printf("Port %d doesn't support hairpin "
2494 /* configure port */
2495 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2496 nb_txq + nb_hairpinq,
2499 if (rte_atomic16_cmpset(&(port->port_status),
2500 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2501 printf("Port %d can not be set back "
2502 "to stopped\n", pi);
2503 printf("Fail to configure port %d\n", pi);
2504 /* try to reconfigure port next time */
2505 port->need_reconfig = 1;
2509 if (port->need_reconfig_queues > 0) {
2510 port->need_reconfig_queues = 0;
2511 /* setup tx queues */
2512 for (qi = 0; qi < nb_txq; qi++) {
2513 if ((numa_support) &&
2514 (txring_numa[pi] != NUMA_NO_CONFIG))
2515 diag = rte_eth_tx_queue_setup(pi, qi,
2516 port->nb_tx_desc[qi],
2518 &(port->tx_conf[qi]));
2520 diag = rte_eth_tx_queue_setup(pi, qi,
2521 port->nb_tx_desc[qi],
2523 &(port->tx_conf[qi]));
2528 /* Fail to setup tx queue, return */
2529 if (rte_atomic16_cmpset(&(port->port_status),
2531 RTE_PORT_STOPPED) == 0)
2532 printf("Port %d can not be set back "
2533 "to stopped\n", pi);
2534 printf("Fail to configure port %d tx queues\n",
2536 /* try to reconfigure queues next time */
2537 port->need_reconfig_queues = 1;
2540 for (qi = 0; qi < nb_rxq; qi++) {
2541 /* setup rx queues */
2542 if ((numa_support) &&
2543 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2544 struct rte_mempool * mp =
2546 (rxring_numa[pi], 0);
2548 printf("Failed to setup RX queue:"
2549 "No mempool allocation"
2550 " on the socket %d\n",
2555 diag = rx_queue_setup(pi, qi,
2556 port->nb_rx_desc[qi],
2558 &(port->rx_conf[qi]),
2561 struct rte_mempool *mp =
2563 (port->socket_id, 0);
2565 printf("Failed to setup RX queue:"
2566 "No mempool allocation"
2567 " on the socket %d\n",
2571 diag = rx_queue_setup(pi, qi,
2572 port->nb_rx_desc[qi],
2574 &(port->rx_conf[qi]),
2580 /* Fail to setup rx queue, return */
2581 if (rte_atomic16_cmpset(&(port->port_status),
2583 RTE_PORT_STOPPED) == 0)
2584 printf("Port %d can not be set back "
2585 "to stopped\n", pi);
2586 printf("Fail to configure port %d rx queues\n",
2588 /* try to reconfigure queues next time */
2589 port->need_reconfig_queues = 1;
2592 /* setup hairpin queues */
2593 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2596 configure_rxtx_dump_callbacks(verbose_level);
2598 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2602 "Port %d: Failed to disable Ptype parsing\n",
2610 diag = rte_eth_dev_start(pi);
2612 printf("Fail to start port %d: %s\n", pi,
2613 rte_strerror(-diag));
2615 /* Fail to setup rx queue, return */
2616 if (rte_atomic16_cmpset(&(port->port_status),
2617 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2618 printf("Port %d can not be set back to "
2623 if (rte_atomic16_cmpset(&(port->port_status),
2624 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2625 printf("Port %d can not be set into started\n", pi);
2627 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
2628 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2629 port->eth_addr.addr_bytes[0],
2630 port->eth_addr.addr_bytes[1],
2631 port->eth_addr.addr_bytes[2],
2632 port->eth_addr.addr_bytes[3],
2633 port->eth_addr.addr_bytes[4],
2634 port->eth_addr.addr_bytes[5]);
2636 /* at least one port started, need checking link status */
2637 need_check_link_status = 1;
2642 if (need_check_link_status == 1 && !no_link_check)
2643 check_all_ports_link_status(RTE_PORT_ALL);
2644 else if (need_check_link_status == 0)
2645 printf("Please stop the ports first\n");
2647 if (hairpin_mode & 0xf) {
2651 /* bind all started hairpin ports */
2652 for (i = 0; i < cfg_pi; i++) {
2654 /* bind current Tx to all peer Rx */
2655 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2656 RTE_MAX_ETHPORTS, 1);
2659 for (j = 0; j < peer_pi; j++) {
2660 if (!port_is_started(peer_pl[j]))
2662 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2664 printf("Error during binding hairpin"
2665 " Tx port %u to %u: %s\n",
2667 rte_strerror(-diag));
2671 /* bind all peer Tx to current Rx */
2672 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2673 RTE_MAX_ETHPORTS, 0);
2676 for (j = 0; j < peer_pi; j++) {
2677 if (!port_is_started(peer_pl[j]))
2679 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2681 printf("Error during binding hairpin"
2682 " Tx port %u to %u: %s\n",
2684 rte_strerror(-diag));
2696 stop_port(portid_t pid)
2699 struct rte_port *port;
2700 int need_check_link_status = 0;
2701 portid_t peer_pl[RTE_MAX_ETHPORTS];
2704 if (port_id_is_invalid(pid, ENABLED_WARN))
2707 printf("Stopping ports...\n");
2709 RTE_ETH_FOREACH_DEV(pi) {
2710 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2713 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2714 printf("Please remove port %d from forwarding configuration.\n", pi);
2718 if (port_is_bonding_slave(pi)) {
2719 printf("Please remove port %d from bonded device.\n", pi);
2724 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2725 RTE_PORT_HANDLING) == 0)
2728 if (hairpin_mode & 0xf) {
2731 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2732 /* unbind all peer Tx from current Rx */
2733 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2734 RTE_MAX_ETHPORTS, 0);
2737 for (j = 0; j < peer_pi; j++) {
2738 if (!port_is_started(peer_pl[j]))
2740 rte_eth_hairpin_unbind(peer_pl[j], pi);
2744 if (port->flow_list)
2745 port_flow_flush(pi);
2747 if (rte_eth_dev_stop(pi) != 0)
2748 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2751 if (rte_atomic16_cmpset(&(port->port_status),
2752 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2753 printf("Port %d can not be set into stopped\n", pi);
2754 need_check_link_status = 1;
2756 if (need_check_link_status && !no_link_check)
2757 check_all_ports_link_status(RTE_PORT_ALL);
2763 remove_invalid_ports_in(portid_t *array, portid_t *total)
2766 portid_t new_total = 0;
2768 for (i = 0; i < *total; i++)
2769 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2770 array[new_total] = array[i];
2777 remove_invalid_ports(void)
2779 remove_invalid_ports_in(ports_ids, &nb_ports);
2780 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2781 nb_cfg_ports = nb_fwd_ports;
2785 close_port(portid_t pid)
2788 struct rte_port *port;
2790 if (port_id_is_invalid(pid, ENABLED_WARN))
2793 printf("Closing ports...\n");
2795 RTE_ETH_FOREACH_DEV(pi) {
2796 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2799 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2800 printf("Please remove port %d from forwarding configuration.\n", pi);
2804 if (port_is_bonding_slave(pi)) {
2805 printf("Please remove port %d from bonded device.\n", pi);
2810 if (rte_atomic16_cmpset(&(port->port_status),
2811 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2812 printf("Port %d is already closed\n", pi);
2816 port_flow_flush(pi);
2817 rte_eth_dev_close(pi);
2820 remove_invalid_ports();
2825 reset_port(portid_t pid)
2829 struct rte_port *port;
2831 if (port_id_is_invalid(pid, ENABLED_WARN))
2834 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2835 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2836 printf("Can not reset port(s), please stop port(s) first.\n");
2840 printf("Resetting ports...\n");
2842 RTE_ETH_FOREACH_DEV(pi) {
2843 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2846 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2847 printf("Please remove port %d from forwarding "
2848 "configuration.\n", pi);
2852 if (port_is_bonding_slave(pi)) {
2853 printf("Please remove port %d from bonded device.\n",
2858 diag = rte_eth_dev_reset(pi);
2861 port->need_reconfig = 1;
2862 port->need_reconfig_queues = 1;
2864 printf("Failed to reset port %d. diag=%d\n", pi, diag);
2872 attach_port(char *identifier)
2875 struct rte_dev_iterator iterator;
2877 printf("Attaching a new port...\n");
2879 if (identifier == NULL) {
2880 printf("Invalid parameters are specified\n");
2884 if (rte_dev_probe(identifier) < 0) {
2885 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2889 /* first attach mode: event */
2890 if (setup_on_probe_event) {
2891 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2892 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2893 if (ports[pi].port_status == RTE_PORT_HANDLING &&
2894 ports[pi].need_setup != 0)
2895 setup_attached_port(pi);
2899 /* second attach mode: iterator */
2900 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2901 /* setup ports matching the devargs used for probing */
2902 if (port_is_forwarding(pi))
2903 continue; /* port was already attached before */
2904 setup_attached_port(pi);
2909 setup_attached_port(portid_t pi)
2911 unsigned int socket_id;
2914 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2915 /* if socket_id is invalid, set to the first available socket. */
2916 if (check_socket_id(socket_id) < 0)
2917 socket_id = socket_ids[0];
2918 reconfig(pi, socket_id);
2919 ret = rte_eth_promiscuous_enable(pi);
2921 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2922 pi, rte_strerror(-ret));
2924 ports_ids[nb_ports++] = pi;
2925 fwd_ports_ids[nb_fwd_ports++] = pi;
2926 nb_cfg_ports = nb_fwd_ports;
2927 ports[pi].need_setup = 0;
2928 ports[pi].port_status = RTE_PORT_STOPPED;
2930 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2935 detach_device(struct rte_device *dev)
2940 printf("Device already removed\n");
2944 printf("Removing a device...\n");
2946 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2947 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2948 if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2949 printf("Port %u not stopped\n", sibling);
2952 port_flow_flush(sibling);
2956 if (rte_dev_remove(dev) < 0) {
2957 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2960 remove_invalid_ports();
2962 printf("Device is detached\n");
2963 printf("Now total ports is %d\n", nb_ports);
2969 detach_port_device(portid_t port_id)
2972 struct rte_eth_dev_info dev_info;
2974 if (port_id_is_invalid(port_id, ENABLED_WARN))
2977 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2978 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2979 printf("Port not stopped\n");
2982 printf("Port was not closed\n");
2985 ret = eth_dev_info_get_print_err(port_id, &dev_info);
2988 "Failed to get device info for port %d, not detaching\n",
2992 detach_device(dev_info.device);
2996 detach_devargs(char *identifier)
2998 struct rte_dev_iterator iterator;
2999 struct rte_devargs da;
3002 printf("Removing a device...\n");
3004 memset(&da, 0, sizeof(da));
3005 if (rte_devargs_parsef(&da, "%s", identifier)) {
3006 printf("cannot parse identifier\n");
3010 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3011 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3012 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3013 printf("Port %u not stopped\n", port_id);
3014 rte_eth_iterator_cleanup(&iterator);
3015 rte_devargs_reset(&da);
3018 port_flow_flush(port_id);
3022 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3023 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3024 da.name, da.bus->name);
3025 rte_devargs_reset(&da);
3029 remove_invalid_ports();
3031 printf("Device %s is detached\n", identifier);
3032 printf("Now total ports is %d\n", nb_ports);
3034 rte_devargs_reset(&da);
3045 stop_packet_forwarding();
3047 #ifndef RTE_EXEC_ENV_WINDOWS
3048 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3050 if (mp_alloc_type == MP_ALLOC_ANON)
3051 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3056 if (ports != NULL) {
3058 RTE_ETH_FOREACH_DEV(pt_id) {
3059 printf("\nStopping port %d...\n", pt_id);
3063 RTE_ETH_FOREACH_DEV(pt_id) {
3064 printf("\nShutting down port %d...\n", pt_id);
3071 ret = rte_dev_event_monitor_stop();
3074 "fail to stop device event monitor.");
3078 ret = rte_dev_event_callback_unregister(NULL,
3079 dev_event_callback, NULL);
3082 "fail to unregister device event callback.\n");
3086 ret = rte_dev_hotplug_handle_disable();
3089 "fail to disable hotplug handling.\n");
3093 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3095 rte_mempool_free(mempools[i]);
3098 printf("\nBye...\n");
3101 typedef void (*cmd_func_t)(void);
3102 struct pmd_test_command {
3103 const char *cmd_name;
3104 cmd_func_t cmd_func;
3107 /* Check the link status of all ports in up to 9s, and print them finally */
3109 check_all_ports_link_status(uint32_t port_mask)
3111 #define CHECK_INTERVAL 100 /* 100ms */
3112 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3114 uint8_t count, all_ports_up, print_flag = 0;
3115 struct rte_eth_link link;
3117 char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3119 printf("Checking link statuses...\n");
3121 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3123 RTE_ETH_FOREACH_DEV(portid) {
3124 if ((port_mask & (1 << portid)) == 0)
3126 memset(&link, 0, sizeof(link));
3127 ret = rte_eth_link_get_nowait(portid, &link);
3130 if (print_flag == 1)
3131 printf("Port %u link get failed: %s\n",
3132 portid, rte_strerror(-ret));
3135 /* print link status if flag set */
3136 if (print_flag == 1) {
3137 rte_eth_link_to_str(link_status,
3138 sizeof(link_status), &link);
3139 printf("Port %d %s\n", portid, link_status);
3142 /* clear all_ports_up flag if any link down */
3143 if (link.link_status == ETH_LINK_DOWN) {
3148 /* after finally printing all link status, get out */
3149 if (print_flag == 1)
3152 if (all_ports_up == 0) {
3154 rte_delay_ms(CHECK_INTERVAL);
3157 /* set the print_flag if all ports up or timeout */
3158 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3168 rmv_port_callback(void *arg)
3170 int need_to_start = 0;
3171 int org_no_link_check = no_link_check;
3172 portid_t port_id = (intptr_t)arg;
3173 struct rte_eth_dev_info dev_info;
3176 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3178 if (!test_done && port_is_forwarding(port_id)) {
3180 stop_packet_forwarding();
3184 no_link_check = org_no_link_check;
3186 close_port(port_id);
3187 ret = eth_dev_info_get_print_err(port_id, &dev_info);
3190 "Failed to get device info for port %d, not detaching\n",
3193 detach_device(dev_info.device); /* might be already removed or have more ports */
3195 start_packet_forwarding(0);
3198 /* This function is used by the interrupt thread */
3200 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3203 RTE_SET_USED(param);
3204 RTE_SET_USED(ret_param);
3206 if (type >= RTE_ETH_EVENT_MAX) {
3207 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3208 port_id, __func__, type);
3210 } else if (event_print_mask & (UINT32_C(1) << type)) {
3211 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3212 eth_event_desc[type]);
3217 case RTE_ETH_EVENT_NEW:
3218 ports[port_id].need_setup = 1;
3219 ports[port_id].port_status = RTE_PORT_HANDLING;
3221 case RTE_ETH_EVENT_INTR_RMV:
3222 if (port_id_is_invalid(port_id, DISABLED_WARN))
3224 if (rte_eal_alarm_set(100000,
3225 rmv_port_callback, (void *)(intptr_t)port_id))
3226 fprintf(stderr, "Could not set up deferred device removal\n");
3228 case RTE_ETH_EVENT_DESTROY:
3229 ports[port_id].port_status = RTE_PORT_CLOSED;
3230 printf("Port %u is closed\n", port_id);
3239 register_eth_event_callback(void)
3242 enum rte_eth_event_type event;
3244 for (event = RTE_ETH_EVENT_UNKNOWN;
3245 event < RTE_ETH_EVENT_MAX; event++) {
3246 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3251 TESTPMD_LOG(ERR, "Failed to register callback for "
3252 "%s event\n", eth_event_desc[event]);
3260 /* This function is used by the interrupt thread */
3262 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3263 __rte_unused void *arg)
3268 if (type >= RTE_DEV_EVENT_MAX) {
3269 fprintf(stderr, "%s called upon invalid event %d\n",
3275 case RTE_DEV_EVENT_REMOVE:
3276 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3278 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3280 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3285 * Because the user's callback is invoked in eal interrupt
3286 * callback, the interrupt callback need to be finished before
3287 * it can be unregistered when detaching device. So finish
3288 * callback soon and use a deferred removal to detach device
3289 * is need. It is a workaround, once the device detaching be
3290 * moved into the eal in the future, the deferred removal could
3293 if (rte_eal_alarm_set(100000,
3294 rmv_port_callback, (void *)(intptr_t)port_id))
3296 "Could not set up deferred device removal\n");
3298 case RTE_DEV_EVENT_ADD:
3299 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3301 /* TODO: After finish kernel driver binding,
3302 * begin to attach port.
3311 rxtx_port_config(struct rte_port *port)
3316 for (qid = 0; qid < nb_rxq; qid++) {
3317 offloads = port->rx_conf[qid].offloads;
3318 port->rx_conf[qid] = port->dev_info.default_rxconf;
3320 port->rx_conf[qid].offloads = offloads;
3322 /* Check if any Rx parameters have been passed */
3323 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3324 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3326 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3327 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3329 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3330 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3332 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3333 port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3335 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3336 port->rx_conf[qid].rx_drop_en = rx_drop_en;
3338 port->nb_rx_desc[qid] = nb_rxd;
3341 for (qid = 0; qid < nb_txq; qid++) {
3342 offloads = port->tx_conf[qid].offloads;
3343 port->tx_conf[qid] = port->dev_info.default_txconf;
3345 port->tx_conf[qid].offloads = offloads;
3347 /* Check if any Tx parameters have been passed */
3348 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3349 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3351 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3352 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3354 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3355 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3357 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3358 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3360 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3361 port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3363 port->nb_tx_desc[qid] = nb_txd;
3368 * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3369 * MTU is also aligned if JUMBO_FRAME offload is not set.
3371 * port->dev_info should be set before calling this function.
3373 * return 0 on success, negative on error
3376 update_jumbo_frame_offload(portid_t portid)
3378 struct rte_port *port = &ports[portid];
3379 uint32_t eth_overhead;
3380 uint64_t rx_offloads;
3384 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3385 if (port->dev_info.max_mtu != UINT16_MAX &&
3386 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3387 eth_overhead = port->dev_info.max_rx_pktlen -
3388 port->dev_info.max_mtu;
3390 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3392 rx_offloads = port->dev_conf.rxmode.offloads;
3394 /* Default config value is 0 to use PMD specific overhead */
3395 if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3396 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3398 if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3399 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3402 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3403 printf("Frame size (%u) is not supported by port %u\n",
3404 port->dev_conf.rxmode.max_rx_pkt_len,
3408 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3412 if (rx_offloads != port->dev_conf.rxmode.offloads) {
3415 port->dev_conf.rxmode.offloads = rx_offloads;
3417 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3418 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3420 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3422 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3426 /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3427 * if unset do it here
3429 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3430 ret = rte_eth_dev_set_mtu(portid,
3431 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3433 printf("Failed to set MTU to %u for port %u\n",
3434 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3442 init_port_config(void)
3445 struct rte_port *port;
3448 RTE_ETH_FOREACH_DEV(pid) {
3450 port->dev_conf.fdir_conf = fdir_conf;
3452 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3457 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3458 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3459 rss_hf & port->dev_info.flow_type_rss_offloads;
3461 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3462 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3465 if (port->dcb_flag == 0) {
3466 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3467 port->dev_conf.rxmode.mq_mode =
3468 (enum rte_eth_rx_mq_mode)
3469 (rx_mq_mode & ETH_MQ_RX_RSS);
3471 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3474 rxtx_port_config(port);
3476 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3480 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3481 rte_pmd_ixgbe_bypass_init(pid);
3484 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3485 port->dev_conf.intr_conf.lsc = 1;
3486 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3487 port->dev_conf.intr_conf.rmv = 1;
3491 void set_port_slave_flag(portid_t slave_pid)
3493 struct rte_port *port;
3495 port = &ports[slave_pid];
3496 port->slave_flag = 1;
3499 void clear_port_slave_flag(portid_t slave_pid)
3501 struct rte_port *port;
3503 port = &ports[slave_pid];
3504 port->slave_flag = 0;
3507 uint8_t port_is_bonding_slave(portid_t slave_pid)
3509 struct rte_port *port;
3510 struct rte_eth_dev_info dev_info;
3513 port = &ports[slave_pid];
3514 ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3517 "Failed to get device info for port id %d,"
3518 "cannot determine if the port is a bonded slave",
3522 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3527 const uint16_t vlan_tags[] = {
3528 0, 1, 2, 3, 4, 5, 6, 7,
3529 8, 9, 10, 11, 12, 13, 14, 15,
3530 16, 17, 18, 19, 20, 21, 22, 23,
3531 24, 25, 26, 27, 28, 29, 30, 31
3535 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3536 enum dcb_mode_enable dcb_mode,
3537 enum rte_eth_nb_tcs num_tcs,
3542 struct rte_eth_rss_conf rss_conf;
3545 * Builds up the correct configuration for dcb+vt based on the vlan tags array
3546 * given above, and the number of traffic classes available for use.
3548 if (dcb_mode == DCB_VT_ENABLED) {
3549 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3550 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3551 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3552 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3554 /* VMDQ+DCB RX and TX configurations */
3555 vmdq_rx_conf->enable_default_pool = 0;
3556 vmdq_rx_conf->default_pool = 0;
3557 vmdq_rx_conf->nb_queue_pools =
3558 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3559 vmdq_tx_conf->nb_queue_pools =
3560 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3562 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3563 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3564 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3565 vmdq_rx_conf->pool_map[i].pools =
3566 1 << (i % vmdq_rx_conf->nb_queue_pools);
3568 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3569 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3570 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3573 /* set DCB mode of RX and TX of multiple queues */
3574 eth_conf->rxmode.mq_mode =
3575 (enum rte_eth_rx_mq_mode)
3576 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3577 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3579 struct rte_eth_dcb_rx_conf *rx_conf =
3580 ð_conf->rx_adv_conf.dcb_rx_conf;
3581 struct rte_eth_dcb_tx_conf *tx_conf =
3582 ð_conf->tx_adv_conf.dcb_tx_conf;
3584 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3586 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3590 rx_conf->nb_tcs = num_tcs;
3591 tx_conf->nb_tcs = num_tcs;
3593 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3594 rx_conf->dcb_tc[i] = i % num_tcs;
3595 tx_conf->dcb_tc[i] = i % num_tcs;
3598 eth_conf->rxmode.mq_mode =
3599 (enum rte_eth_rx_mq_mode)
3600 (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3601 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3602 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3606 eth_conf->dcb_capability_en =
3607 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3609 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3615 init_port_dcb_config(portid_t pid,
3616 enum dcb_mode_enable dcb_mode,
3617 enum rte_eth_nb_tcs num_tcs,
3620 struct rte_eth_conf port_conf;
3621 struct rte_port *rte_port;
3625 rte_port = &ports[pid];
3627 memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3629 port_conf.rxmode = rte_port->dev_conf.rxmode;
3630 port_conf.txmode = rte_port->dev_conf.txmode;
3632 /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3633 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3636 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3638 /* re-configure the device . */
3639 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3643 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3647 /* If dev_info.vmdq_pool_base is greater than 0,
3648 * the queue id of vmdq pools is started after pf queues.
3650 if (dcb_mode == DCB_VT_ENABLED &&
3651 rte_port->dev_info.vmdq_pool_base > 0) {
3652 printf("VMDQ_DCB multi-queue mode is nonsensical"
3653 " for port %d.", pid);
3657 /* Assume the ports in testpmd have the same dcb capability
3658 * and has the same number of rxq and txq in dcb mode
3660 if (dcb_mode == DCB_VT_ENABLED) {
3661 if (rte_port->dev_info.max_vfs > 0) {
3662 nb_rxq = rte_port->dev_info.nb_rx_queues;
3663 nb_txq = rte_port->dev_info.nb_tx_queues;
3665 nb_rxq = rte_port->dev_info.max_rx_queues;
3666 nb_txq = rte_port->dev_info.max_tx_queues;
3669 /*if vt is disabled, use all pf queues */
3670 if (rte_port->dev_info.vmdq_pool_base == 0) {
3671 nb_rxq = rte_port->dev_info.max_rx_queues;
3672 nb_txq = rte_port->dev_info.max_tx_queues;
3674 nb_rxq = (queueid_t)num_tcs;
3675 nb_txq = (queueid_t)num_tcs;
3679 rx_free_thresh = 64;
3681 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3683 rxtx_port_config(rte_port);
3685 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3686 for (i = 0; i < RTE_DIM(vlan_tags); i++)
3687 rx_vft_set(pid, vlan_tags[i], 1);
3689 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3693 rte_port->dcb_flag = 1;
3695 /* Enter DCB configuration status */
3706 /* Configuration of Ethernet ports. */
3707 ports = rte_zmalloc("testpmd: ports",
3708 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3709 RTE_CACHE_LINE_SIZE);
3710 if (ports == NULL) {
3711 rte_exit(EXIT_FAILURE,
3712 "rte_zmalloc(%d struct rte_port) failed\n",
3715 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3716 LIST_INIT(&ports[i].flow_tunnel_list);
3717 /* Initialize ports NUMA structures */
3718 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3719 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3720 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3734 const char clr[] = { 27, '[', '2', 'J', '\0' };
3735 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3737 /* Clear screen and move to top left */
3738 printf("%s%s", clr, top_left);
3740 printf("\nPort statistics ====================================");
3741 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3742 nic_stats_display(fwd_ports_ids[i]);
3748 signal_handler(int signum)
3750 if (signum == SIGINT || signum == SIGTERM) {
3751 printf("\nSignal %d received, preparing to exit...\n",
3753 #ifdef RTE_LIB_PDUMP
3754 /* uninitialize packet capture framework */
3757 #ifdef RTE_LIB_LATENCYSTATS
3758 if (latencystats_enabled != 0)
3759 rte_latencystats_uninit();
3762 /* Set flag to indicate the force termination. */
3764 /* exit with the expected status */
3765 #ifndef RTE_EXEC_ENV_WINDOWS
3766 signal(signum, SIG_DFL);
3767 kill(getpid(), signum);
3773 main(int argc, char** argv)
3780 signal(SIGINT, signal_handler);
3781 signal(SIGTERM, signal_handler);
3783 testpmd_logtype = rte_log_register("testpmd");
3784 if (testpmd_logtype < 0)
3785 rte_exit(EXIT_FAILURE, "Cannot register log type");
3786 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3788 diag = rte_eal_init(argc, argv);
3790 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3791 rte_strerror(rte_errno));
3793 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3794 rte_exit(EXIT_FAILURE,
3795 "Secondary process type not supported.\n");
3797 ret = register_eth_event_callback();
3799 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3801 #ifdef RTE_LIB_PDUMP
3802 /* initialize packet capture framework */
3807 RTE_ETH_FOREACH_DEV(port_id) {
3808 ports_ids[count] = port_id;
3811 nb_ports = (portid_t) count;
3813 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3815 /* allocate port structures, and init them */
3818 set_def_fwd_config();
3820 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3821 "Check the core mask argument\n");
3823 /* Bitrate/latency stats disabled by default */
3824 #ifdef RTE_LIB_BITRATESTATS
3825 bitrate_enabled = 0;
3827 #ifdef RTE_LIB_LATENCYSTATS
3828 latencystats_enabled = 0;
3831 /* on FreeBSD, mlockall() is disabled by default */
3832 #ifdef RTE_EXEC_ENV_FREEBSD
3841 launch_args_parse(argc, argv);
3843 #ifndef RTE_EXEC_ENV_WINDOWS
3844 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3845 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3850 if (tx_first && interactive)
3851 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3852 "interactive mode.\n");
3854 if (tx_first && lsc_interrupt) {
3855 printf("Warning: lsc_interrupt needs to be off when "
3856 " using tx_first. Disabling.\n");
3860 if (!nb_rxq && !nb_txq)
3861 printf("Warning: Either rx or tx queues should be non-zero\n");
3863 if (nb_rxq > 1 && nb_rxq > nb_txq)
3864 printf("Warning: nb_rxq=%d enables RSS configuration, "
3865 "but nb_txq=%d will prevent to fully test it.\n",
3871 ret = rte_dev_hotplug_handle_enable();
3874 "fail to enable hotplug handling.");
3878 ret = rte_dev_event_monitor_start();
3881 "fail to start device event monitoring.");
3885 ret = rte_dev_event_callback_register(NULL,
3886 dev_event_callback, NULL);
3889 "fail to register device event callback\n");
3894 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3895 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3897 /* set all ports to promiscuous mode by default */
3898 RTE_ETH_FOREACH_DEV(port_id) {
3899 ret = rte_eth_promiscuous_enable(port_id);
3901 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3902 port_id, rte_strerror(-ret));
3905 /* Init metrics library */
3906 rte_metrics_init(rte_socket_id());
3908 #ifdef RTE_LIB_LATENCYSTATS
3909 if (latencystats_enabled != 0) {
3910 int ret = rte_latencystats_init(1, NULL);
3912 printf("Warning: latencystats init()"
3913 " returned error %d\n", ret);
3914 printf("Latencystats running on lcore %d\n",
3915 latencystats_lcore_id);
3919 /* Setup bitrate stats */
3920 #ifdef RTE_LIB_BITRATESTATS
3921 if (bitrate_enabled != 0) {
3922 bitrate_data = rte_stats_bitrate_create();
3923 if (bitrate_data == NULL)
3924 rte_exit(EXIT_FAILURE,
3925 "Could not allocate bitrate data.\n");
3926 rte_stats_bitrate_reg(bitrate_data);
3930 #ifdef RTE_LIB_CMDLINE
3931 if (strlen(cmdline_filename) != 0)
3932 cmdline_read_from_file(cmdline_filename);
3934 if (interactive == 1) {
3936 printf("Start automatic packet forwarding\n");
3937 start_packet_forwarding(0);
3949 printf("No commandline core given, start packet forwarding\n");
3950 start_packet_forwarding(tx_first);
3951 if (stats_period != 0) {
3952 uint64_t prev_time = 0, cur_time, diff_time = 0;
3953 uint64_t timer_period;
3955 /* Convert to number of cycles */
3956 timer_period = stats_period * rte_get_timer_hz();
3958 while (f_quit == 0) {
3959 cur_time = rte_get_timer_cycles();
3960 diff_time += cur_time - prev_time;
3962 if (diff_time >= timer_period) {
3964 /* Reset the timer */
3967 /* Sleep to avoid unnecessary checks */
3968 prev_time = cur_time;
3969 rte_delay_us_sleep(US_PER_S);
3973 printf("Press enter to exit\n");
3974 rc = read(0, &c, 1);
3980 ret = rte_eal_cleanup();
3982 rte_exit(EXIT_FAILURE,
3983 "EAL cleanup failed: %s\n", strerror(-ret));
3985 return EXIT_SUCCESS;