1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
13 #include <sys/types.h>
17 #include <sys/queue.h>
24 #include <rte_common.h>
25 #include <rte_errno.h>
26 #include <rte_byteorder.h>
28 #include <rte_debug.h>
29 #include <rte_cycles.h>
30 #include <rte_memory.h>
31 #include <rte_memcpy.h>
32 #include <rte_launch.h>
34 #include <rte_alarm.h>
35 #include <rte_per_lcore.h>
36 #include <rte_lcore.h>
37 #include <rte_atomic.h>
38 #include <rte_branch_prediction.h>
39 #include <rte_mempool.h>
40 #include <rte_malloc.h>
42 #include <rte_mbuf_pool_ops.h>
43 #include <rte_interrupts.h>
45 #include <rte_ether.h>
46 #include <rte_ethdev.h>
48 #include <rte_string_fns.h>
49 #ifdef RTE_LIBRTE_IXGBE_PMD
50 #include <rte_pmd_ixgbe.h>
52 #ifdef RTE_LIBRTE_PDUMP
53 #include <rte_pdump.h>
56 #include <rte_metrics.h>
57 #ifdef RTE_LIBRTE_BITRATE
58 #include <rte_bitrate.h>
60 #ifdef RTE_LIBRTE_LATENCY_STATS
61 #include <rte_latencystats.h>
67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
68 #define HUGE_FLAG (0x40000)
70 #define HUGE_FLAG MAP_HUGETLB
73 #ifndef MAP_HUGE_SHIFT
74 /* older kernels (or FreeBSD) will not have this define */
75 #define HUGE_SHIFT (26)
77 #define HUGE_SHIFT MAP_HUGE_SHIFT
80 #define EXTMEM_HEAP_NAME "extmem"
81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
83 uint16_t verbose_level = 0; /**< Silent by default. */
84 int testpmd_logtype; /**< Log type for testpmd logs */
86 /* use master core for command line ? */
87 uint8_t interactive = 0;
88 uint8_t auto_start = 0;
90 char cmdline_filename[PATH_MAX] = {0};
93 * NUMA support configuration.
94 * When set, the NUMA support attempts to dispatch the allocation of the
95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
96 * probed ports among the CPU sockets 0 and 1.
97 * Otherwise, all memory is allocated from CPU socket 0.
99 uint8_t numa_support = 1; /**< numa enabled by default */
102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is
105 uint8_t socket_num = UMA_NO_CONFIG;
108 * Select mempool allocation type:
109 * - native: use regular DPDK memory
110 * - anon: use regular DPDK memory to create mempool, but populate using
111 * anonymous memory (may not be IOVA-contiguous)
112 * - xmem: use externally allocated hugepage memory
114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
117 * Store specified sockets on which memory pool to be used by ports
120 uint8_t port_numa[RTE_MAX_ETHPORTS];
123 * Store specified sockets on which RX ring to be used by ports
126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
129 * Store specified sockets on which TX ring to be used by ports
132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
135 * Record the Ethernet address of peer target ports to which packets are
137 * Must be instantiated with the ethernet addresses of peer traffic generator
140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
141 portid_t nb_peer_eth_addrs = 0;
144 * Probed Target Environment.
146 struct rte_port *ports; /**< For all probed ethernet ports. */
147 portid_t nb_ports; /**< Number of probed ethernet ports. */
148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */
151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
154 * Test Forwarding Configuration.
155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports
158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
160 portid_t nb_cfg_ports; /**< Number of configured ports. */
161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */
163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */
166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */
170 * Forwarding engines.
172 struct fwd_engine * fwd_engines[] = {
182 &five_tuple_swap_fwd_engine,
183 #ifdef RTE_LIBRTE_IEEE1588
184 &ieee1588_fwd_engine,
189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
190 uint16_t mempool_flags;
192 struct fwd_config cur_fwd_config;
193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
194 uint32_t retry_enabled;
195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
198 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
199 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if
200 * specified on command-line. */
201 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
204 * In container, it cannot terminate the process which running with 'stats-period'
205 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
210 * Configuration of packet segments used by the "txonly" processing engine.
212 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
213 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
214 TXONLY_DEF_PACKET_LEN,
216 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
218 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
219 /**< Split policy for packets to TX. */
221 uint8_t txonly_multi_flow;
222 /**< Whether multiple flows are generated in TXONLY mode. */
224 uint32_t tx_pkt_times_inter;
225 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
227 uint32_t tx_pkt_times_intra;
228 /**< Timings for send scheduling in TXONLY mode, time between packets. */
230 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
231 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
233 /* current configuration is in DCB or not,0 means it is not in DCB mode */
234 uint8_t dcb_config = 0;
236 /* Whether the dcb is in testing status */
237 uint8_t dcb_test = 0;
240 * Configurable number of RX/TX queues.
242 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
243 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
244 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
247 * Configurable number of RX/TX ring descriptors.
248 * Defaults are supplied by drivers via ethdev.
250 #define RTE_TEST_RX_DESC_DEFAULT 0
251 #define RTE_TEST_TX_DESC_DEFAULT 0
252 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
253 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
255 #define RTE_PMD_PARAM_UNSET -1
257 * Configurable values of RX and TX ring threshold registers.
260 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
261 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
262 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
264 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
265 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
266 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
269 * Configurable value of RX free threshold.
271 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
274 * Configurable value of RX drop enable.
276 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
279 * Configurable value of TX free threshold.
281 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
284 * Configurable value of TX RS bit threshold.
286 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
289 * Configurable value of buffered packets before sending.
291 uint16_t noisy_tx_sw_bufsz;
294 * Configurable value of packet buffer timeout.
296 uint16_t noisy_tx_sw_buf_flush_time;
299 * Configurable value for size of VNF internal memory area
300 * used for simulating noisy neighbour behaviour
302 uint64_t noisy_lkup_mem_sz;
305 * Configurable value of number of random writes done in
306 * VNF simulation memory area.
308 uint64_t noisy_lkup_num_writes;
311 * Configurable value of number of random reads done in
312 * VNF simulation memory area.
314 uint64_t noisy_lkup_num_reads;
317 * Configurable value of number of random reads/writes done in
318 * VNF simulation memory area.
320 uint64_t noisy_lkup_num_reads_writes;
323 * Receive Side Scaling (RSS) configuration.
325 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
328 * Port topology configuration
330 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
333 * Avoids to flush all the RX streams before starts forwarding.
335 uint8_t no_flush_rx = 0; /* flush by default */
338 * Flow API isolated mode.
340 uint8_t flow_isolate_all;
343 * Avoids to check link status when starting/stopping a port.
345 uint8_t no_link_check = 0; /* check by default */
348 * Don't automatically start all ports in interactive mode.
350 uint8_t no_device_start = 0;
353 * Enable link status change notification
355 uint8_t lsc_interrupt = 1; /* enabled by default */
358 * Enable device removal notification.
360 uint8_t rmv_interrupt = 1; /* enabled by default */
362 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
364 /* After attach, port setup is called on event or by iterator */
365 bool setup_on_probe_event = true;
367 /* Clear ptypes on port initialization. */
368 uint8_t clear_ptypes = true;
370 /* Pretty printing of ethdev events */
371 static const char * const eth_event_desc[] = {
372 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
373 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
374 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
375 [RTE_ETH_EVENT_INTR_RESET] = "reset",
376 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
377 [RTE_ETH_EVENT_IPSEC] = "IPsec",
378 [RTE_ETH_EVENT_MACSEC] = "MACsec",
379 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
380 [RTE_ETH_EVENT_NEW] = "device probed",
381 [RTE_ETH_EVENT_DESTROY] = "device released",
382 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
383 [RTE_ETH_EVENT_MAX] = NULL,
387 * Display or mask ether events
388 * Default to all events except VF_MBOX
390 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
391 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
392 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
393 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
394 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
395 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
396 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
397 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
399 * Decide if all memory are locked for performance.
404 * NIC bypass mode configuration options.
407 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
408 /* The NIC bypass watchdog timeout. */
409 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
413 #ifdef RTE_LIBRTE_LATENCY_STATS
416 * Set when latency stats is enabled in the commandline
418 uint8_t latencystats_enabled;
421 * Lcore ID to serive latency statistics.
423 lcoreid_t latencystats_lcore_id = -1;
428 * Ethernet device configuration.
430 struct rte_eth_rxmode rx_mode = {
431 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
432 /**< Default maximum frame length. */
435 struct rte_eth_txmode tx_mode = {
436 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
439 struct rte_fdir_conf fdir_conf = {
440 .mode = RTE_FDIR_MODE_NONE,
441 .pballoc = RTE_FDIR_PBALLOC_64K,
442 .status = RTE_FDIR_REPORT_STATUS,
444 .vlan_tci_mask = 0xFFEF,
446 .src_ip = 0xFFFFFFFF,
447 .dst_ip = 0xFFFFFFFF,
450 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
451 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
453 .src_port_mask = 0xFFFF,
454 .dst_port_mask = 0xFFFF,
455 .mac_addr_byte_mask = 0xFF,
456 .tunnel_type_mask = 1,
457 .tunnel_id_mask = 0xFFFFFFFF,
462 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
464 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
465 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
467 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
468 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
470 uint16_t nb_tx_queue_stats_mappings = 0;
471 uint16_t nb_rx_queue_stats_mappings = 0;
474 * Display zero values by default for xstats
476 uint8_t xstats_hide_zero;
478 unsigned int num_sockets = 0;
479 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
481 #ifdef RTE_LIBRTE_BITRATE
482 /* Bitrate statistics */
483 struct rte_stats_bitrates *bitrate_data;
484 lcoreid_t bitrate_lcore_id;
485 uint8_t bitrate_enabled;
488 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
489 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
492 * hexadecimal bitmask of RX mq mode can be enabled.
494 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
496 /* Forward function declarations */
497 static void setup_attached_port(portid_t pi);
498 static void map_port_queue_stats_mapping_registers(portid_t pi,
499 struct rte_port *port);
500 static void check_all_ports_link_status(uint32_t port_mask);
501 static int eth_event_callback(portid_t port_id,
502 enum rte_eth_event_type type,
503 void *param, void *ret_param);
504 static void dev_event_callback(const char *device_name,
505 enum rte_dev_event_type type,
509 * Check if all the ports are started.
510 * If yes, return positive value. If not, return zero.
512 static int all_ports_started(void);
514 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
515 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
517 /* Holds the registered mbuf dynamic flags names. */
518 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
521 * Helper function to check if socket is already discovered.
522 * If yes, return positive value. If not, return zero.
525 new_socket_id(unsigned int socket_id)
529 for (i = 0; i < num_sockets; i++) {
530 if (socket_ids[i] == socket_id)
537 * Setup default configuration.
540 set_default_fwd_lcores_config(void)
544 unsigned int sock_num;
547 for (i = 0; i < RTE_MAX_LCORE; i++) {
548 if (!rte_lcore_is_enabled(i))
550 sock_num = rte_lcore_to_socket_id(i);
551 if (new_socket_id(sock_num)) {
552 if (num_sockets >= RTE_MAX_NUMA_NODES) {
553 rte_exit(EXIT_FAILURE,
554 "Total sockets greater than %u\n",
557 socket_ids[num_sockets++] = sock_num;
559 if (i == rte_get_master_lcore())
561 fwd_lcores_cpuids[nb_lc++] = i;
563 nb_lcores = (lcoreid_t) nb_lc;
564 nb_cfg_lcores = nb_lcores;
569 set_def_peer_eth_addrs(void)
573 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
574 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
575 peer_eth_addrs[i].addr_bytes[5] = i;
580 set_default_fwd_ports_config(void)
585 RTE_ETH_FOREACH_DEV(pt_id) {
586 fwd_ports_ids[i++] = pt_id;
588 /* Update sockets info according to the attached device */
589 int socket_id = rte_eth_dev_socket_id(pt_id);
590 if (socket_id >= 0 && new_socket_id(socket_id)) {
591 if (num_sockets >= RTE_MAX_NUMA_NODES) {
592 rte_exit(EXIT_FAILURE,
593 "Total sockets greater than %u\n",
596 socket_ids[num_sockets++] = socket_id;
600 nb_cfg_ports = nb_ports;
601 nb_fwd_ports = nb_ports;
605 set_def_fwd_config(void)
607 set_default_fwd_lcores_config();
608 set_def_peer_eth_addrs();
609 set_default_fwd_ports_config();
612 /* extremely pessimistic estimation of memory required to create a mempool */
614 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
616 unsigned int n_pages, mbuf_per_pg, leftover;
617 uint64_t total_mem, mbuf_mem, obj_sz;
619 /* there is no good way to predict how much space the mempool will
620 * occupy because it will allocate chunks on the fly, and some of those
621 * will come from default DPDK memory while some will come from our
622 * external memory, so just assume 128MB will be enough for everyone.
624 uint64_t hdr_mem = 128 << 20;
626 /* account for possible non-contiguousness */
627 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
629 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
633 mbuf_per_pg = pgsz / obj_sz;
634 leftover = (nb_mbufs % mbuf_per_pg) > 0;
635 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
637 mbuf_mem = n_pages * pgsz;
639 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
641 if (total_mem > SIZE_MAX) {
642 TESTPMD_LOG(ERR, "Memory size too big\n");
645 *out = (size_t)total_mem;
651 pagesz_flags(uint64_t page_sz)
653 /* as per mmap() manpage, all page sizes are log2 of page size
654 * shifted by MAP_HUGE_SHIFT
656 int log2 = rte_log2_u64(page_sz);
658 return (log2 << HUGE_SHIFT);
662 alloc_mem(size_t memsz, size_t pgsz, bool huge)
667 /* allocate anonymous hugepages */
668 flags = MAP_ANONYMOUS | MAP_PRIVATE;
670 flags |= HUGE_FLAG | pagesz_flags(pgsz);
672 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
673 if (addr == MAP_FAILED)
679 struct extmem_param {
683 rte_iova_t *iova_table;
684 unsigned int iova_table_len;
688 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
691 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
692 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */
693 unsigned int cur_page, n_pages, pgsz_idx;
694 size_t mem_sz, cur_pgsz;
695 rte_iova_t *iovas = NULL;
699 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
700 /* skip anything that is too big */
701 if (pgsizes[pgsz_idx] > SIZE_MAX)
704 cur_pgsz = pgsizes[pgsz_idx];
706 /* if we were told not to allocate hugepages, override */
708 cur_pgsz = sysconf(_SC_PAGESIZE);
710 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
712 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
716 /* allocate our memory */
717 addr = alloc_mem(mem_sz, cur_pgsz, huge);
719 /* if we couldn't allocate memory with a specified page size,
720 * that doesn't mean we can't do it with other page sizes, so
726 /* store IOVA addresses for every page in this memory area */
727 n_pages = mem_sz / cur_pgsz;
729 iovas = malloc(sizeof(*iovas) * n_pages);
732 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
735 /* lock memory if it's not huge pages */
739 /* populate IOVA addresses */
740 for (cur_page = 0; cur_page < n_pages; cur_page++) {
745 offset = cur_pgsz * cur_page;
746 cur = RTE_PTR_ADD(addr, offset);
748 /* touch the page before getting its IOVA */
749 *(volatile char *)cur = 0;
751 iova = rte_mem_virt2iova(cur);
753 iovas[cur_page] = iova;
758 /* if we couldn't allocate anything */
764 param->pgsz = cur_pgsz;
765 param->iova_table = iovas;
766 param->iova_table_len = n_pages;
773 munmap(addr, mem_sz);
779 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
781 struct extmem_param param;
784 memset(¶m, 0, sizeof(param));
786 /* check if our heap exists */
787 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
789 /* create our heap */
790 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
792 TESTPMD_LOG(ERR, "Cannot create heap\n");
797 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
799 TESTPMD_LOG(ERR, "Cannot create memory area\n");
803 /* we now have a valid memory area, so add it to heap */
804 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
805 param.addr, param.len, param.iova_table,
806 param.iova_table_len, param.pgsz);
808 /* when using VFIO, memory is automatically mapped for DMA by EAL */
810 /* not needed any more */
811 free(param.iova_table);
814 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
815 munmap(param.addr, param.len);
821 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
827 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
828 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
833 RTE_ETH_FOREACH_DEV(pid) {
834 struct rte_eth_dev *dev =
835 &rte_eth_devices[pid];
837 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
841 "unable to DMA unmap addr 0x%p "
843 memhdr->addr, dev->data->name);
846 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
849 "unable to un-register addr 0x%p\n", memhdr->addr);
854 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
855 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
858 size_t page_size = sysconf(_SC_PAGESIZE);
861 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
865 "unable to register addr 0x%p\n", memhdr->addr);
868 RTE_ETH_FOREACH_DEV(pid) {
869 struct rte_eth_dev *dev =
870 &rte_eth_devices[pid];
872 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
876 "unable to DMA map addr 0x%p "
878 memhdr->addr, dev->data->name);
884 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
885 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
887 struct rte_pktmbuf_extmem *xmem;
888 unsigned int ext_num, zone_num, elt_num;
891 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
892 elt_num = EXTBUF_ZONE_SIZE / elt_size;
893 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
895 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
897 TESTPMD_LOG(ERR, "Cannot allocate memory for "
898 "external buffer descriptors\n");
902 for (ext_num = 0; ext_num < zone_num; ext_num++) {
903 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
904 const struct rte_memzone *mz;
905 char mz_name[RTE_MEMZONE_NAMESIZE];
908 ret = snprintf(mz_name, sizeof(mz_name),
909 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
910 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
911 errno = ENAMETOOLONG;
915 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
917 RTE_MEMZONE_IOVA_CONTIG |
919 RTE_MEMZONE_SIZE_HINT_ONLY,
923 * The caller exits on external buffer creation
924 * error, so there is no need to free memzones.
930 xseg->buf_ptr = mz->addr;
931 xseg->buf_iova = mz->iova;
932 xseg->buf_len = EXTBUF_ZONE_SIZE;
933 xseg->elt_size = elt_size;
935 if (ext_num == 0 && xmem != NULL) {
944 * Configuration initialisation done once at init time.
946 static struct rte_mempool *
947 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
948 unsigned int socket_id)
950 char pool_name[RTE_MEMPOOL_NAMESIZE];
951 struct rte_mempool *rte_mp = NULL;
954 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
955 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
958 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
959 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
961 switch (mp_alloc_type) {
962 case MP_ALLOC_NATIVE:
964 /* wrapper to rte_mempool_create() */
965 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
966 rte_mbuf_best_mempool_ops());
967 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
968 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
973 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
974 mb_size, (unsigned int) mb_mempool_cache,
975 sizeof(struct rte_pktmbuf_pool_private),
976 socket_id, mempool_flags);
980 if (rte_mempool_populate_anon(rte_mp) == 0) {
981 rte_mempool_free(rte_mp);
985 rte_pktmbuf_pool_init(rte_mp, NULL);
986 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
987 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
991 case MP_ALLOC_XMEM_HUGE:
994 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
996 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
997 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1000 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1001 if (heap_socket < 0)
1002 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1004 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1005 rte_mbuf_best_mempool_ops());
1006 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1007 mb_mempool_cache, 0, mbuf_seg_size,
1013 struct rte_pktmbuf_extmem *ext_mem;
1014 unsigned int ext_num;
1016 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1017 socket_id, pool_name, &ext_mem);
1019 rte_exit(EXIT_FAILURE,
1020 "Can't create pinned data buffers\n");
1022 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1023 rte_mbuf_best_mempool_ops());
1024 rte_mp = rte_pktmbuf_pool_create_extbuf
1025 (pool_name, nb_mbuf, mb_mempool_cache,
1026 0, mbuf_seg_size, socket_id,
1033 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1038 if (rte_mp == NULL) {
1039 rte_exit(EXIT_FAILURE,
1040 "Creation of mbuf pool for socket %u failed: %s\n",
1041 socket_id, rte_strerror(rte_errno));
1042 } else if (verbose_level > 0) {
1043 rte_mempool_dump(stdout, rte_mp);
1049 * Check given socket id is valid or not with NUMA mode,
1050 * if valid, return 0, else return -1
1053 check_socket_id(const unsigned int socket_id)
1055 static int warning_once = 0;
1057 if (new_socket_id(socket_id)) {
1058 if (!warning_once && numa_support)
1059 printf("Warning: NUMA should be configured manually by"
1060 " using --port-numa-config and"
1061 " --ring-numa-config parameters along with"
1070 * Get the allowed maximum number of RX queues.
1071 * *pid return the port id which has minimal value of
1072 * max_rx_queues in all ports.
1075 get_allowed_max_nb_rxq(portid_t *pid)
1077 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1078 bool max_rxq_valid = false;
1080 struct rte_eth_dev_info dev_info;
1082 RTE_ETH_FOREACH_DEV(pi) {
1083 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1086 max_rxq_valid = true;
1087 if (dev_info.max_rx_queues < allowed_max_rxq) {
1088 allowed_max_rxq = dev_info.max_rx_queues;
1092 return max_rxq_valid ? allowed_max_rxq : 0;
1096 * Check input rxq is valid or not.
1097 * If input rxq is not greater than any of maximum number
1098 * of RX queues of all ports, it is valid.
1099 * if valid, return 0, else return -1
1102 check_nb_rxq(queueid_t rxq)
1104 queueid_t allowed_max_rxq;
1107 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1108 if (rxq > allowed_max_rxq) {
1109 printf("Fail: input rxq (%u) can't be greater "
1110 "than max_rx_queues (%u) of port %u\n",
1120 * Get the allowed maximum number of TX queues.
1121 * *pid return the port id which has minimal value of
1122 * max_tx_queues in all ports.
1125 get_allowed_max_nb_txq(portid_t *pid)
1127 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1128 bool max_txq_valid = false;
1130 struct rte_eth_dev_info dev_info;
1132 RTE_ETH_FOREACH_DEV(pi) {
1133 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1136 max_txq_valid = true;
1137 if (dev_info.max_tx_queues < allowed_max_txq) {
1138 allowed_max_txq = dev_info.max_tx_queues;
1142 return max_txq_valid ? allowed_max_txq : 0;
1146 * Check input txq is valid or not.
1147 * If input txq is not greater than any of maximum number
1148 * of TX queues of all ports, it is valid.
1149 * if valid, return 0, else return -1
1152 check_nb_txq(queueid_t txq)
1154 queueid_t allowed_max_txq;
1157 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1158 if (txq > allowed_max_txq) {
1159 printf("Fail: input txq (%u) can't be greater "
1160 "than max_tx_queues (%u) of port %u\n",
1170 * Get the allowed maximum number of RXDs of every rx queue.
1171 * *pid return the port id which has minimal value of
1172 * max_rxd in all queues of all ports.
1175 get_allowed_max_nb_rxd(portid_t *pid)
1177 uint16_t allowed_max_rxd = UINT16_MAX;
1179 struct rte_eth_dev_info dev_info;
1181 RTE_ETH_FOREACH_DEV(pi) {
1182 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1185 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1186 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1190 return allowed_max_rxd;
1194 * Get the allowed minimal number of RXDs of every rx queue.
1195 * *pid return the port id which has minimal value of
1196 * min_rxd in all queues of all ports.
1199 get_allowed_min_nb_rxd(portid_t *pid)
1201 uint16_t allowed_min_rxd = 0;
1203 struct rte_eth_dev_info dev_info;
1205 RTE_ETH_FOREACH_DEV(pi) {
1206 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1209 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1210 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1215 return allowed_min_rxd;
1219 * Check input rxd is valid or not.
1220 * If input rxd is not greater than any of maximum number
1221 * of RXDs of every Rx queues and is not less than any of
1222 * minimal number of RXDs of every Rx queues, it is valid.
1223 * if valid, return 0, else return -1
1226 check_nb_rxd(queueid_t rxd)
1228 uint16_t allowed_max_rxd;
1229 uint16_t allowed_min_rxd;
1232 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1233 if (rxd > allowed_max_rxd) {
1234 printf("Fail: input rxd (%u) can't be greater "
1235 "than max_rxds (%u) of port %u\n",
1242 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1243 if (rxd < allowed_min_rxd) {
1244 printf("Fail: input rxd (%u) can't be less "
1245 "than min_rxds (%u) of port %u\n",
1256 * Get the allowed maximum number of TXDs of every rx queues.
1257 * *pid return the port id which has minimal value of
1258 * max_txd in every tx queue.
1261 get_allowed_max_nb_txd(portid_t *pid)
1263 uint16_t allowed_max_txd = UINT16_MAX;
1265 struct rte_eth_dev_info dev_info;
1267 RTE_ETH_FOREACH_DEV(pi) {
1268 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1271 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1272 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1276 return allowed_max_txd;
1280 * Get the allowed maximum number of TXDs of every tx queues.
1281 * *pid return the port id which has minimal value of
1282 * min_txd in every tx queue.
1285 get_allowed_min_nb_txd(portid_t *pid)
1287 uint16_t allowed_min_txd = 0;
1289 struct rte_eth_dev_info dev_info;
1291 RTE_ETH_FOREACH_DEV(pi) {
1292 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1295 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1296 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1301 return allowed_min_txd;
1305 * Check input txd is valid or not.
1306 * If input txd is not greater than any of maximum number
1307 * of TXDs of every Rx queues, it is valid.
1308 * if valid, return 0, else return -1
1311 check_nb_txd(queueid_t txd)
1313 uint16_t allowed_max_txd;
1314 uint16_t allowed_min_txd;
1317 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1318 if (txd > allowed_max_txd) {
1319 printf("Fail: input txd (%u) can't be greater "
1320 "than max_txds (%u) of port %u\n",
1327 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1328 if (txd < allowed_min_txd) {
1329 printf("Fail: input txd (%u) can't be less "
1330 "than min_txds (%u) of port %u\n",
1341 * Get the allowed maximum number of hairpin queues.
1342 * *pid return the port id which has minimal value of
1343 * max_hairpin_queues in all ports.
1346 get_allowed_max_nb_hairpinq(portid_t *pid)
1348 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1350 struct rte_eth_hairpin_cap cap;
1352 RTE_ETH_FOREACH_DEV(pi) {
1353 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1357 if (cap.max_nb_queues < allowed_max_hairpinq) {
1358 allowed_max_hairpinq = cap.max_nb_queues;
1362 return allowed_max_hairpinq;
1366 * Check input hairpin is valid or not.
1367 * If input hairpin is not greater than any of maximum number
1368 * of hairpin queues of all ports, it is valid.
1369 * if valid, return 0, else return -1
1372 check_nb_hairpinq(queueid_t hairpinq)
1374 queueid_t allowed_max_hairpinq;
1377 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1378 if (hairpinq > allowed_max_hairpinq) {
1379 printf("Fail: input hairpin (%u) can't be greater "
1380 "than max_hairpin_queues (%u) of port %u\n",
1381 hairpinq, allowed_max_hairpinq, pid);
1391 struct rte_port *port;
1392 struct rte_mempool *mbp;
1393 unsigned int nb_mbuf_per_pool;
1395 uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1396 struct rte_gro_param gro_param;
1403 memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1405 /* Configuration of logical cores. */
1406 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1407 sizeof(struct fwd_lcore *) * nb_lcores,
1408 RTE_CACHE_LINE_SIZE);
1409 if (fwd_lcores == NULL) {
1410 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1411 "failed\n", nb_lcores);
1413 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1414 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1415 sizeof(struct fwd_lcore),
1416 RTE_CACHE_LINE_SIZE);
1417 if (fwd_lcores[lc_id] == NULL) {
1418 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1421 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1424 RTE_ETH_FOREACH_DEV(pid) {
1426 /* Apply default TxRx configuration for all ports */
1427 port->dev_conf.txmode = tx_mode;
1428 port->dev_conf.rxmode = rx_mode;
1430 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1432 rte_exit(EXIT_FAILURE,
1433 "rte_eth_dev_info_get() failed\n");
1435 if (!(port->dev_info.tx_offload_capa &
1436 DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1437 port->dev_conf.txmode.offloads &=
1438 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1440 if (port_numa[pid] != NUMA_NO_CONFIG)
1441 port_per_socket[port_numa[pid]]++;
1443 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1446 * if socket_id is invalid,
1447 * set to the first available socket.
1449 if (check_socket_id(socket_id) < 0)
1450 socket_id = socket_ids[0];
1451 port_per_socket[socket_id]++;
1455 /* Apply Rx offloads configuration */
1456 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1457 port->rx_conf[k].offloads =
1458 port->dev_conf.rxmode.offloads;
1459 /* Apply Tx offloads configuration */
1460 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1461 port->tx_conf[k].offloads =
1462 port->dev_conf.txmode.offloads;
1464 /* set flag to initialize port/queue */
1465 port->need_reconfig = 1;
1466 port->need_reconfig_queues = 1;
1467 port->tx_metadata = 0;
1469 /* Check for maximum number of segments per MTU. Accordingly
1470 * update the mbuf data size.
1472 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1473 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1474 data_size = rx_mode.max_rx_pkt_len /
1475 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1477 if ((data_size + RTE_PKTMBUF_HEADROOM) >
1479 mbuf_data_size = data_size +
1480 RTE_PKTMBUF_HEADROOM;
1487 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1491 * Create pools of mbuf.
1492 * If NUMA support is disabled, create a single pool of mbuf in
1493 * socket 0 memory by default.
1494 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1496 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1497 * nb_txd can be configured at run time.
1499 if (param_total_num_mbufs)
1500 nb_mbuf_per_pool = param_total_num_mbufs;
1502 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1503 (nb_lcores * mb_mempool_cache) +
1504 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1505 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1511 for (i = 0; i < num_sockets; i++)
1512 mempools[i] = mbuf_pool_create(mbuf_data_size,
1516 if (socket_num == UMA_NO_CONFIG)
1517 mempools[0] = mbuf_pool_create(mbuf_data_size,
1518 nb_mbuf_per_pool, 0);
1520 mempools[socket_num] = mbuf_pool_create
1528 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1529 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1531 * Records which Mbuf pool to use by each logical core, if needed.
1533 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1534 mbp = mbuf_pool_find(
1535 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1538 mbp = mbuf_pool_find(0);
1539 fwd_lcores[lc_id]->mbp = mbp;
1540 /* initialize GSO context */
1541 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1542 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1543 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1544 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1546 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1549 /* Configuration of packet forwarding streams. */
1550 if (init_fwd_streams() < 0)
1551 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1555 /* create a gro context for each lcore */
1556 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1557 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1558 gro_param.max_item_per_flow = MAX_PKT_BURST;
1559 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1560 gro_param.socket_id = rte_lcore_to_socket_id(
1561 fwd_lcores_cpuids[lc_id]);
1562 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1563 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1564 rte_exit(EXIT_FAILURE,
1565 "rte_gro_ctx_create() failed\n");
1572 reconfig(portid_t new_port_id, unsigned socket_id)
1574 struct rte_port *port;
1577 /* Reconfiguration of Ethernet ports. */
1578 port = &ports[new_port_id];
1580 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1584 /* set flag to initialize port/queue */
1585 port->need_reconfig = 1;
1586 port->need_reconfig_queues = 1;
1587 port->socket_id = socket_id;
1594 init_fwd_streams(void)
1597 struct rte_port *port;
1598 streamid_t sm_id, nb_fwd_streams_new;
1601 /* set socket id according to numa or not */
1602 RTE_ETH_FOREACH_DEV(pid) {
1604 if (nb_rxq > port->dev_info.max_rx_queues) {
1605 printf("Fail: nb_rxq(%d) is greater than "
1606 "max_rx_queues(%d)\n", nb_rxq,
1607 port->dev_info.max_rx_queues);
1610 if (nb_txq > port->dev_info.max_tx_queues) {
1611 printf("Fail: nb_txq(%d) is greater than "
1612 "max_tx_queues(%d)\n", nb_txq,
1613 port->dev_info.max_tx_queues);
1617 if (port_numa[pid] != NUMA_NO_CONFIG)
1618 port->socket_id = port_numa[pid];
1620 port->socket_id = rte_eth_dev_socket_id(pid);
1623 * if socket_id is invalid,
1624 * set to the first available socket.
1626 if (check_socket_id(port->socket_id) < 0)
1627 port->socket_id = socket_ids[0];
1631 if (socket_num == UMA_NO_CONFIG)
1632 port->socket_id = 0;
1634 port->socket_id = socket_num;
1638 q = RTE_MAX(nb_rxq, nb_txq);
1640 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1643 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1644 if (nb_fwd_streams_new == nb_fwd_streams)
1647 if (fwd_streams != NULL) {
1648 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1649 if (fwd_streams[sm_id] == NULL)
1651 rte_free(fwd_streams[sm_id]);
1652 fwd_streams[sm_id] = NULL;
1654 rte_free(fwd_streams);
1659 nb_fwd_streams = nb_fwd_streams_new;
1660 if (nb_fwd_streams) {
1661 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1662 sizeof(struct fwd_stream *) * nb_fwd_streams,
1663 RTE_CACHE_LINE_SIZE);
1664 if (fwd_streams == NULL)
1665 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1666 " (struct fwd_stream *)) failed\n",
1669 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1670 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1671 " struct fwd_stream", sizeof(struct fwd_stream),
1672 RTE_CACHE_LINE_SIZE);
1673 if (fwd_streams[sm_id] == NULL)
1674 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1675 "(struct fwd_stream) failed\n");
1682 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1684 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1686 uint64_t total_burst, sburst;
1688 uint64_t burst_stats[4];
1689 uint16_t pktnb_stats[4];
1691 int burst_percent[4], sburstp;
1695 * First compute the total number of packet bursts and the
1696 * two highest numbers of bursts of the same number of packets.
1698 memset(&burst_stats, 0x0, sizeof(burst_stats));
1699 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1701 /* Show stats for 0 burst size always */
1702 total_burst = pbs->pkt_burst_spread[0];
1703 burst_stats[0] = pbs->pkt_burst_spread[0];
1706 /* Find the next 2 burst sizes with highest occurrences. */
1707 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1708 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1713 total_burst += nb_burst;
1715 if (nb_burst > burst_stats[1]) {
1716 burst_stats[2] = burst_stats[1];
1717 pktnb_stats[2] = pktnb_stats[1];
1718 burst_stats[1] = nb_burst;
1719 pktnb_stats[1] = nb_pkt;
1720 } else if (nb_burst > burst_stats[2]) {
1721 burst_stats[2] = nb_burst;
1722 pktnb_stats[2] = nb_pkt;
1725 if (total_burst == 0)
1728 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1729 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1731 printf("%d%% of other]\n", 100 - sburstp);
1735 sburst += burst_stats[i];
1736 if (sburst == total_burst) {
1737 printf("%d%% of %d pkts]\n",
1738 100 - sburstp, (int) pktnb_stats[i]);
1743 (double)burst_stats[i] / total_burst * 100;
1744 printf("%d%% of %d pkts + ",
1745 burst_percent[i], (int) pktnb_stats[i]);
1746 sburstp += burst_percent[i];
1749 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1752 fwd_stream_stats_display(streamid_t stream_id)
1754 struct fwd_stream *fs;
1755 static const char *fwd_top_stats_border = "-------";
1757 fs = fwd_streams[stream_id];
1758 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1759 (fs->fwd_dropped == 0))
1761 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1762 "TX Port=%2d/Queue=%2d %s\n",
1763 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1764 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1765 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1766 " TX-dropped: %-14"PRIu64,
1767 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1769 /* if checksum mode */
1770 if (cur_fwd_eng == &csum_fwd_engine) {
1771 printf(" RX- bad IP checksum: %-14"PRIu64
1772 " Rx- bad L4 checksum: %-14"PRIu64
1773 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1774 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1775 fs->rx_bad_outer_l4_csum);
1780 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1781 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1782 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1787 fwd_stats_display(void)
1789 static const char *fwd_stats_border = "----------------------";
1790 static const char *acc_stats_border = "+++++++++++++++";
1792 struct fwd_stream *rx_stream;
1793 struct fwd_stream *tx_stream;
1794 uint64_t tx_dropped;
1795 uint64_t rx_bad_ip_csum;
1796 uint64_t rx_bad_l4_csum;
1797 uint64_t rx_bad_outer_l4_csum;
1798 } ports_stats[RTE_MAX_ETHPORTS];
1799 uint64_t total_rx_dropped = 0;
1800 uint64_t total_tx_dropped = 0;
1801 uint64_t total_rx_nombuf = 0;
1802 struct rte_eth_stats stats;
1803 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1804 uint64_t fwd_cycles = 0;
1806 uint64_t total_recv = 0;
1807 uint64_t total_xmit = 0;
1808 struct rte_port *port;
1813 memset(ports_stats, 0, sizeof(ports_stats));
1815 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1816 struct fwd_stream *fs = fwd_streams[sm_id];
1818 if (cur_fwd_config.nb_fwd_streams >
1819 cur_fwd_config.nb_fwd_ports) {
1820 fwd_stream_stats_display(sm_id);
1822 ports_stats[fs->tx_port].tx_stream = fs;
1823 ports_stats[fs->rx_port].rx_stream = fs;
1826 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1828 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1829 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1830 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1831 fs->rx_bad_outer_l4_csum;
1833 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1834 fwd_cycles += fs->core_cycles;
1837 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1840 pt_id = fwd_ports_ids[i];
1841 port = &ports[pt_id];
1843 rte_eth_stats_get(pt_id, &stats);
1844 stats.ipackets -= port->stats.ipackets;
1845 stats.opackets -= port->stats.opackets;
1846 stats.ibytes -= port->stats.ibytes;
1847 stats.obytes -= port->stats.obytes;
1848 stats.imissed -= port->stats.imissed;
1849 stats.oerrors -= port->stats.oerrors;
1850 stats.rx_nombuf -= port->stats.rx_nombuf;
1852 total_recv += stats.ipackets;
1853 total_xmit += stats.opackets;
1854 total_rx_dropped += stats.imissed;
1855 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1856 total_tx_dropped += stats.oerrors;
1857 total_rx_nombuf += stats.rx_nombuf;
1859 printf("\n %s Forward statistics for port %-2d %s\n",
1860 fwd_stats_border, pt_id, fwd_stats_border);
1862 if (!port->rx_queue_stats_mapping_enabled &&
1863 !port->tx_queue_stats_mapping_enabled) {
1864 printf(" RX-packets: %-14"PRIu64
1865 " RX-dropped: %-14"PRIu64
1866 "RX-total: %-"PRIu64"\n",
1867 stats.ipackets, stats.imissed,
1868 stats.ipackets + stats.imissed);
1870 if (cur_fwd_eng == &csum_fwd_engine)
1871 printf(" Bad-ipcsum: %-14"PRIu64
1872 " Bad-l4csum: %-14"PRIu64
1873 "Bad-outer-l4csum: %-14"PRIu64"\n",
1874 ports_stats[pt_id].rx_bad_ip_csum,
1875 ports_stats[pt_id].rx_bad_l4_csum,
1876 ports_stats[pt_id].rx_bad_outer_l4_csum);
1877 if (stats.ierrors + stats.rx_nombuf > 0) {
1878 printf(" RX-error: %-"PRIu64"\n",
1880 printf(" RX-nombufs: %-14"PRIu64"\n",
1884 printf(" TX-packets: %-14"PRIu64
1885 " TX-dropped: %-14"PRIu64
1886 "TX-total: %-"PRIu64"\n",
1887 stats.opackets, ports_stats[pt_id].tx_dropped,
1888 stats.opackets + ports_stats[pt_id].tx_dropped);
1890 printf(" RX-packets: %14"PRIu64
1891 " RX-dropped:%14"PRIu64
1892 " RX-total:%14"PRIu64"\n",
1893 stats.ipackets, stats.imissed,
1894 stats.ipackets + stats.imissed);
1896 if (cur_fwd_eng == &csum_fwd_engine)
1897 printf(" Bad-ipcsum:%14"PRIu64
1898 " Bad-l4csum:%14"PRIu64
1899 " Bad-outer-l4csum: %-14"PRIu64"\n",
1900 ports_stats[pt_id].rx_bad_ip_csum,
1901 ports_stats[pt_id].rx_bad_l4_csum,
1902 ports_stats[pt_id].rx_bad_outer_l4_csum);
1903 if ((stats.ierrors + stats.rx_nombuf) > 0) {
1904 printf(" RX-error:%"PRIu64"\n", stats.ierrors);
1905 printf(" RX-nombufs: %14"PRIu64"\n",
1909 printf(" TX-packets: %14"PRIu64
1910 " TX-dropped:%14"PRIu64
1911 " TX-total:%14"PRIu64"\n",
1912 stats.opackets, ports_stats[pt_id].tx_dropped,
1913 stats.opackets + ports_stats[pt_id].tx_dropped);
1916 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1917 if (ports_stats[pt_id].rx_stream)
1918 pkt_burst_stats_display("RX",
1919 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1920 if (ports_stats[pt_id].tx_stream)
1921 pkt_burst_stats_display("TX",
1922 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1925 if (port->rx_queue_stats_mapping_enabled) {
1927 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1928 printf(" Stats reg %2d RX-packets:%14"PRIu64
1929 " RX-errors:%14"PRIu64
1930 " RX-bytes:%14"PRIu64"\n",
1931 j, stats.q_ipackets[j],
1932 stats.q_errors[j], stats.q_ibytes[j]);
1936 if (port->tx_queue_stats_mapping_enabled) {
1937 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1938 printf(" Stats reg %2d TX-packets:%14"PRIu64
1941 j, stats.q_opackets[j],
1946 printf(" %s--------------------------------%s\n",
1947 fwd_stats_border, fwd_stats_border);
1950 printf("\n %s Accumulated forward statistics for all ports"
1952 acc_stats_border, acc_stats_border);
1953 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1955 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1957 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1958 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1959 if (total_rx_nombuf > 0)
1960 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1961 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
1963 acc_stats_border, acc_stats_border);
1964 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1965 #define CYC_PER_MHZ 1E6
1966 if (total_recv > 0 || total_xmit > 0) {
1967 uint64_t total_pkts = 0;
1968 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1969 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1970 total_pkts = total_xmit;
1972 total_pkts = total_recv;
1974 printf("\n CPU cycles/packet=%.2F (total cycles="
1975 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1977 (double) fwd_cycles / total_pkts,
1978 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1979 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1985 fwd_stats_reset(void)
1991 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1992 pt_id = fwd_ports_ids[i];
1993 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1995 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1996 struct fwd_stream *fs = fwd_streams[sm_id];
2000 fs->fwd_dropped = 0;
2001 fs->rx_bad_ip_csum = 0;
2002 fs->rx_bad_l4_csum = 0;
2003 fs->rx_bad_outer_l4_csum = 0;
2005 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
2006 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2007 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2009 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
2010 fs->core_cycles = 0;
2016 flush_fwd_rx_queues(void)
2018 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2025 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2026 uint64_t timer_period;
2028 /* convert to number of cycles */
2029 timer_period = rte_get_timer_hz(); /* 1 second timeout */
2031 for (j = 0; j < 2; j++) {
2032 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2033 for (rxq = 0; rxq < nb_rxq; rxq++) {
2034 port_id = fwd_ports_ids[rxp];
2036 * testpmd can stuck in the below do while loop
2037 * if rte_eth_rx_burst() always returns nonzero
2038 * packets. So timer is added to exit this loop
2039 * after 1sec timer expiry.
2041 prev_tsc = rte_rdtsc();
2043 nb_rx = rte_eth_rx_burst(port_id, rxq,
2044 pkts_burst, MAX_PKT_BURST);
2045 for (i = 0; i < nb_rx; i++)
2046 rte_pktmbuf_free(pkts_burst[i]);
2048 cur_tsc = rte_rdtsc();
2049 diff_tsc = cur_tsc - prev_tsc;
2050 timer_tsc += diff_tsc;
2051 } while ((nb_rx > 0) &&
2052 (timer_tsc < timer_period));
2056 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2061 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2063 struct fwd_stream **fsm;
2066 #ifdef RTE_LIBRTE_BITRATE
2067 uint64_t tics_per_1sec;
2068 uint64_t tics_datum;
2069 uint64_t tics_current;
2070 uint16_t i, cnt_ports;
2072 cnt_ports = nb_ports;
2073 tics_datum = rte_rdtsc();
2074 tics_per_1sec = rte_get_timer_hz();
2076 fsm = &fwd_streams[fc->stream_idx];
2077 nb_fs = fc->stream_nb;
2079 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2080 (*pkt_fwd)(fsm[sm_id]);
2081 #ifdef RTE_LIBRTE_BITRATE
2082 if (bitrate_enabled != 0 &&
2083 bitrate_lcore_id == rte_lcore_id()) {
2084 tics_current = rte_rdtsc();
2085 if (tics_current - tics_datum >= tics_per_1sec) {
2086 /* Periodic bitrate calculation */
2087 for (i = 0; i < cnt_ports; i++)
2088 rte_stats_bitrate_calc(bitrate_data,
2090 tics_datum = tics_current;
2094 #ifdef RTE_LIBRTE_LATENCY_STATS
2095 if (latencystats_enabled != 0 &&
2096 latencystats_lcore_id == rte_lcore_id())
2097 rte_latencystats_update();
2100 } while (! fc->stopped);
2104 start_pkt_forward_on_core(void *fwd_arg)
2106 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2107 cur_fwd_config.fwd_eng->packet_fwd);
2112 * Run the TXONLY packet forwarding engine to send a single burst of packets.
2113 * Used to start communication flows in network loopback test configurations.
2116 run_one_txonly_burst_on_core(void *fwd_arg)
2118 struct fwd_lcore *fwd_lc;
2119 struct fwd_lcore tmp_lcore;
2121 fwd_lc = (struct fwd_lcore *) fwd_arg;
2122 tmp_lcore = *fwd_lc;
2123 tmp_lcore.stopped = 1;
2124 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2129 * Launch packet forwarding:
2130 * - Setup per-port forwarding context.
2131 * - launch logical cores with their forwarding configuration.
2134 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2136 port_fwd_begin_t port_fwd_begin;
2141 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2142 if (port_fwd_begin != NULL) {
2143 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2144 (*port_fwd_begin)(fwd_ports_ids[i]);
2146 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2147 lc_id = fwd_lcores_cpuids[i];
2148 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2149 fwd_lcores[i]->stopped = 0;
2150 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2151 fwd_lcores[i], lc_id);
2153 printf("launch lcore %u failed - diag=%d\n",
2160 * Launch packet forwarding configuration.
2163 start_packet_forwarding(int with_tx_first)
2165 port_fwd_begin_t port_fwd_begin;
2166 port_fwd_end_t port_fwd_end;
2167 struct rte_port *port;
2171 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2172 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2174 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2175 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2177 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2178 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2179 (!nb_rxq || !nb_txq))
2180 rte_exit(EXIT_FAILURE,
2181 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2182 cur_fwd_eng->fwd_mode_name);
2184 if (all_ports_started() == 0) {
2185 printf("Not all ports were started\n");
2188 if (test_done == 0) {
2189 printf("Packet forwarding already started\n");
2195 for (i = 0; i < nb_fwd_ports; i++) {
2196 pt_id = fwd_ports_ids[i];
2197 port = &ports[pt_id];
2198 if (!port->dcb_flag) {
2199 printf("In DCB mode, all forwarding ports must "
2200 "be configured in this mode.\n");
2204 if (nb_fwd_lcores == 1) {
2205 printf("In DCB mode,the nb forwarding cores "
2206 "should be larger than 1.\n");
2215 flush_fwd_rx_queues();
2217 pkt_fwd_config_display(&cur_fwd_config);
2218 rxtx_config_display();
2221 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2222 pt_id = fwd_ports_ids[i];
2223 port = &ports[pt_id];
2224 map_port_queue_stats_mapping_registers(pt_id, port);
2226 if (with_tx_first) {
2227 port_fwd_begin = tx_only_engine.port_fwd_begin;
2228 if (port_fwd_begin != NULL) {
2229 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2230 (*port_fwd_begin)(fwd_ports_ids[i]);
2232 while (with_tx_first--) {
2233 launch_packet_forwarding(
2234 run_one_txonly_burst_on_core);
2235 rte_eal_mp_wait_lcore();
2237 port_fwd_end = tx_only_engine.port_fwd_end;
2238 if (port_fwd_end != NULL) {
2239 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2240 (*port_fwd_end)(fwd_ports_ids[i]);
2243 launch_packet_forwarding(start_pkt_forward_on_core);
2247 stop_packet_forwarding(void)
2249 port_fwd_end_t port_fwd_end;
2255 printf("Packet forwarding not started\n");
2258 printf("Telling cores to stop...");
2259 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2260 fwd_lcores[lc_id]->stopped = 1;
2261 printf("\nWaiting for lcores to finish...\n");
2262 rte_eal_mp_wait_lcore();
2263 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2264 if (port_fwd_end != NULL) {
2265 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2266 pt_id = fwd_ports_ids[i];
2267 (*port_fwd_end)(pt_id);
2271 fwd_stats_display();
2273 printf("\nDone.\n");
2278 dev_set_link_up(portid_t pid)
2280 if (rte_eth_dev_set_link_up(pid) < 0)
2281 printf("\nSet link up fail.\n");
2285 dev_set_link_down(portid_t pid)
2287 if (rte_eth_dev_set_link_down(pid) < 0)
2288 printf("\nSet link down fail.\n");
2292 all_ports_started(void)
2295 struct rte_port *port;
2297 RTE_ETH_FOREACH_DEV(pi) {
2299 /* Check if there is a port which is not started */
2300 if ((port->port_status != RTE_PORT_STARTED) &&
2301 (port->slave_flag == 0))
2305 /* No port is not started */
2310 port_is_stopped(portid_t port_id)
2312 struct rte_port *port = &ports[port_id];
2314 if ((port->port_status != RTE_PORT_STOPPED) &&
2315 (port->slave_flag == 0))
2321 all_ports_stopped(void)
2325 RTE_ETH_FOREACH_DEV(pi) {
2326 if (!port_is_stopped(pi))
2334 port_is_started(portid_t port_id)
2336 if (port_id_is_invalid(port_id, ENABLED_WARN))
2339 if (ports[port_id].port_status != RTE_PORT_STARTED)
2345 /* Configure the Rx and Tx hairpin queues for the selected port. */
2347 setup_hairpin_queues(portid_t pi)
2350 struct rte_eth_hairpin_conf hairpin_conf = {
2355 struct rte_port *port = &ports[pi];
2357 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2358 hairpin_conf.peers[0].port = pi;
2359 hairpin_conf.peers[0].queue = i + nb_rxq;
2360 diag = rte_eth_tx_hairpin_queue_setup
2361 (pi, qi, nb_txd, &hairpin_conf);
2366 /* Fail to setup rx queue, return */
2367 if (rte_atomic16_cmpset(&(port->port_status),
2369 RTE_PORT_STOPPED) == 0)
2370 printf("Port %d can not be set back "
2371 "to stopped\n", pi);
2372 printf("Fail to configure port %d hairpin "
2374 /* try to reconfigure queues next time */
2375 port->need_reconfig_queues = 1;
2378 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2379 hairpin_conf.peers[0].port = pi;
2380 hairpin_conf.peers[0].queue = i + nb_txq;
2381 diag = rte_eth_rx_hairpin_queue_setup
2382 (pi, qi, nb_rxd, &hairpin_conf);
2387 /* Fail to setup rx queue, return */
2388 if (rte_atomic16_cmpset(&(port->port_status),
2390 RTE_PORT_STOPPED) == 0)
2391 printf("Port %d can not be set back "
2392 "to stopped\n", pi);
2393 printf("Fail to configure port %d hairpin "
2395 /* try to reconfigure queues next time */
2396 port->need_reconfig_queues = 1;
2403 start_port(portid_t pid)
2405 int diag, need_check_link_status = -1;
2408 struct rte_port *port;
2409 struct rte_ether_addr mac_addr;
2410 struct rte_eth_hairpin_cap cap;
2412 if (port_id_is_invalid(pid, ENABLED_WARN))
2417 RTE_ETH_FOREACH_DEV(pi) {
2418 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2421 need_check_link_status = 0;
2423 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2424 RTE_PORT_HANDLING) == 0) {
2425 printf("Port %d is now not stopped\n", pi);
2429 if (port->need_reconfig > 0) {
2430 port->need_reconfig = 0;
2432 if (flow_isolate_all) {
2433 int ret = port_flow_isolate(pi, 1);
2435 printf("Failed to apply isolated"
2436 " mode on port %d\n", pi);
2440 configure_rxtx_dump_callbacks(0);
2441 printf("Configuring Port %d (socket %u)\n", pi,
2443 if (nb_hairpinq > 0 &&
2444 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2445 printf("Port %d doesn't support hairpin "
2449 /* configure port */
2450 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2451 nb_txq + nb_hairpinq,
2454 if (rte_atomic16_cmpset(&(port->port_status),
2455 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2456 printf("Port %d can not be set back "
2457 "to stopped\n", pi);
2458 printf("Fail to configure port %d\n", pi);
2459 /* try to reconfigure port next time */
2460 port->need_reconfig = 1;
2464 if (port->need_reconfig_queues > 0) {
2465 port->need_reconfig_queues = 0;
2466 /* setup tx queues */
2467 for (qi = 0; qi < nb_txq; qi++) {
2468 if ((numa_support) &&
2469 (txring_numa[pi] != NUMA_NO_CONFIG))
2470 diag = rte_eth_tx_queue_setup(pi, qi,
2471 port->nb_tx_desc[qi],
2473 &(port->tx_conf[qi]));
2475 diag = rte_eth_tx_queue_setup(pi, qi,
2476 port->nb_tx_desc[qi],
2478 &(port->tx_conf[qi]));
2483 /* Fail to setup tx queue, return */
2484 if (rte_atomic16_cmpset(&(port->port_status),
2486 RTE_PORT_STOPPED) == 0)
2487 printf("Port %d can not be set back "
2488 "to stopped\n", pi);
2489 printf("Fail to configure port %d tx queues\n",
2491 /* try to reconfigure queues next time */
2492 port->need_reconfig_queues = 1;
2495 for (qi = 0; qi < nb_rxq; qi++) {
2496 /* setup rx queues */
2497 if ((numa_support) &&
2498 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2499 struct rte_mempool * mp =
2500 mbuf_pool_find(rxring_numa[pi]);
2502 printf("Failed to setup RX queue:"
2503 "No mempool allocation"
2504 " on the socket %d\n",
2509 diag = rte_eth_rx_queue_setup(pi, qi,
2510 port->nb_rx_desc[qi],
2512 &(port->rx_conf[qi]),
2515 struct rte_mempool *mp =
2516 mbuf_pool_find(port->socket_id);
2518 printf("Failed to setup RX queue:"
2519 "No mempool allocation"
2520 " on the socket %d\n",
2524 diag = rte_eth_rx_queue_setup(pi, qi,
2525 port->nb_rx_desc[qi],
2527 &(port->rx_conf[qi]),
2533 /* Fail to setup rx queue, return */
2534 if (rte_atomic16_cmpset(&(port->port_status),
2536 RTE_PORT_STOPPED) == 0)
2537 printf("Port %d can not be set back "
2538 "to stopped\n", pi);
2539 printf("Fail to configure port %d rx queues\n",
2541 /* try to reconfigure queues next time */
2542 port->need_reconfig_queues = 1;
2545 /* setup hairpin queues */
2546 if (setup_hairpin_queues(pi) != 0)
2549 configure_rxtx_dump_callbacks(verbose_level);
2551 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2555 "Port %d: Failed to disable Ptype parsing\n",
2560 if (rte_eth_dev_start(pi) < 0) {
2561 printf("Fail to start port %d\n", pi);
2563 /* Fail to setup rx queue, return */
2564 if (rte_atomic16_cmpset(&(port->port_status),
2565 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2566 printf("Port %d can not be set back to "
2571 if (rte_atomic16_cmpset(&(port->port_status),
2572 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2573 printf("Port %d can not be set into started\n", pi);
2575 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2576 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2577 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2578 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2579 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2581 /* at least one port started, need checking link status */
2582 need_check_link_status = 1;
2585 if (need_check_link_status == 1 && !no_link_check)
2586 check_all_ports_link_status(RTE_PORT_ALL);
2587 else if (need_check_link_status == 0)
2588 printf("Please stop the ports first\n");
2595 stop_port(portid_t pid)
2598 struct rte_port *port;
2599 int need_check_link_status = 0;
2606 if (port_id_is_invalid(pid, ENABLED_WARN))
2609 printf("Stopping ports...\n");
2611 RTE_ETH_FOREACH_DEV(pi) {
2612 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2615 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2616 printf("Please remove port %d from forwarding configuration.\n", pi);
2620 if (port_is_bonding_slave(pi)) {
2621 printf("Please remove port %d from bonded device.\n", pi);
2626 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2627 RTE_PORT_HANDLING) == 0)
2630 rte_eth_dev_stop(pi);
2632 if (rte_atomic16_cmpset(&(port->port_status),
2633 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2634 printf("Port %d can not be set into stopped\n", pi);
2635 need_check_link_status = 1;
2637 if (need_check_link_status && !no_link_check)
2638 check_all_ports_link_status(RTE_PORT_ALL);
2644 remove_invalid_ports_in(portid_t *array, portid_t *total)
2647 portid_t new_total = 0;
2649 for (i = 0; i < *total; i++)
2650 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2651 array[new_total] = array[i];
2658 remove_invalid_ports(void)
2660 remove_invalid_ports_in(ports_ids, &nb_ports);
2661 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2662 nb_cfg_ports = nb_fwd_ports;
2666 close_port(portid_t pid)
2669 struct rte_port *port;
2671 if (port_id_is_invalid(pid, ENABLED_WARN))
2674 printf("Closing ports...\n");
2676 RTE_ETH_FOREACH_DEV(pi) {
2677 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2680 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2681 printf("Please remove port %d from forwarding configuration.\n", pi);
2685 if (port_is_bonding_slave(pi)) {
2686 printf("Please remove port %d from bonded device.\n", pi);
2691 if (rte_atomic16_cmpset(&(port->port_status),
2692 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2693 printf("Port %d is already closed\n", pi);
2697 if (rte_atomic16_cmpset(&(port->port_status),
2698 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2699 printf("Port %d is now not stopped\n", pi);
2703 if (port->flow_list)
2704 port_flow_flush(pi);
2705 rte_eth_dev_close(pi);
2707 remove_invalid_ports();
2709 if (rte_atomic16_cmpset(&(port->port_status),
2710 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2711 printf("Port %d cannot be set to closed\n", pi);
2718 reset_port(portid_t pid)
2722 struct rte_port *port;
2724 if (port_id_is_invalid(pid, ENABLED_WARN))
2727 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2728 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2729 printf("Can not reset port(s), please stop port(s) first.\n");
2733 printf("Resetting ports...\n");
2735 RTE_ETH_FOREACH_DEV(pi) {
2736 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2739 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2740 printf("Please remove port %d from forwarding "
2741 "configuration.\n", pi);
2745 if (port_is_bonding_slave(pi)) {
2746 printf("Please remove port %d from bonded device.\n",
2751 diag = rte_eth_dev_reset(pi);
2754 port->need_reconfig = 1;
2755 port->need_reconfig_queues = 1;
2757 printf("Failed to reset port %d. diag=%d\n", pi, diag);
2765 attach_port(char *identifier)
2768 struct rte_dev_iterator iterator;
2770 printf("Attaching a new port...\n");
2772 if (identifier == NULL) {
2773 printf("Invalid parameters are specified\n");
2777 if (rte_dev_probe(identifier) < 0) {
2778 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2782 /* first attach mode: event */
2783 if (setup_on_probe_event) {
2784 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2785 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2786 if (ports[pi].port_status == RTE_PORT_HANDLING &&
2787 ports[pi].need_setup != 0)
2788 setup_attached_port(pi);
2792 /* second attach mode: iterator */
2793 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2794 /* setup ports matching the devargs used for probing */
2795 if (port_is_forwarding(pi))
2796 continue; /* port was already attached before */
2797 setup_attached_port(pi);
2802 setup_attached_port(portid_t pi)
2804 unsigned int socket_id;
2807 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2808 /* if socket_id is invalid, set to the first available socket. */
2809 if (check_socket_id(socket_id) < 0)
2810 socket_id = socket_ids[0];
2811 reconfig(pi, socket_id);
2812 ret = rte_eth_promiscuous_enable(pi);
2814 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2815 pi, rte_strerror(-ret));
2817 ports_ids[nb_ports++] = pi;
2818 fwd_ports_ids[nb_fwd_ports++] = pi;
2819 nb_cfg_ports = nb_fwd_ports;
2820 ports[pi].need_setup = 0;
2821 ports[pi].port_status = RTE_PORT_STOPPED;
2823 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2828 detach_device(struct rte_device *dev)
2833 printf("Device already removed\n");
2837 printf("Removing a device...\n");
2839 if (rte_dev_remove(dev) < 0) {
2840 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2843 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2844 /* reset mapping between old ports and removed device */
2845 rte_eth_devices[sibling].device = NULL;
2846 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2847 /* sibling ports are forced to be closed */
2848 ports[sibling].port_status = RTE_PORT_CLOSED;
2849 printf("Port %u is closed\n", sibling);
2853 remove_invalid_ports();
2855 printf("Device is detached\n");
2856 printf("Now total ports is %d\n", nb_ports);
2862 detach_port_device(portid_t port_id)
2864 if (port_id_is_invalid(port_id, ENABLED_WARN))
2867 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2868 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2869 printf("Port not stopped\n");
2872 printf("Port was not closed\n");
2873 if (ports[port_id].flow_list)
2874 port_flow_flush(port_id);
2877 detach_device(rte_eth_devices[port_id].device);
2881 detach_devargs(char *identifier)
2883 struct rte_dev_iterator iterator;
2884 struct rte_devargs da;
2887 printf("Removing a device...\n");
2889 memset(&da, 0, sizeof(da));
2890 if (rte_devargs_parsef(&da, "%s", identifier)) {
2891 printf("cannot parse identifier\n");
2897 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2898 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2899 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2900 printf("Port %u not stopped\n", port_id);
2901 rte_eth_iterator_cleanup(&iterator);
2905 /* sibling ports are forced to be closed */
2906 if (ports[port_id].flow_list)
2907 port_flow_flush(port_id);
2908 ports[port_id].port_status = RTE_PORT_CLOSED;
2909 printf("Port %u is now closed\n", port_id);
2913 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2914 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2915 da.name, da.bus->name);
2919 remove_invalid_ports();
2921 printf("Device %s is detached\n", identifier);
2922 printf("Now total ports is %d\n", nb_ports);
2934 stop_packet_forwarding();
2936 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2938 if (mp_alloc_type == MP_ALLOC_ANON)
2939 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2943 if (ports != NULL) {
2945 RTE_ETH_FOREACH_DEV(pt_id) {
2946 printf("\nStopping port %d...\n", pt_id);
2950 RTE_ETH_FOREACH_DEV(pt_id) {
2951 printf("\nShutting down port %d...\n", pt_id);
2958 ret = rte_dev_event_monitor_stop();
2961 "fail to stop device event monitor.");
2965 ret = rte_dev_event_callback_unregister(NULL,
2966 dev_event_callback, NULL);
2969 "fail to unregister device event callback.\n");
2973 ret = rte_dev_hotplug_handle_disable();
2976 "fail to disable hotplug handling.\n");
2980 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2982 rte_mempool_free(mempools[i]);
2985 printf("\nBye...\n");
2988 typedef void (*cmd_func_t)(void);
2989 struct pmd_test_command {
2990 const char *cmd_name;
2991 cmd_func_t cmd_func;
2994 /* Check the link status of all ports in up to 9s, and print them finally */
2996 check_all_ports_link_status(uint32_t port_mask)
2998 #define CHECK_INTERVAL 100 /* 100ms */
2999 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3001 uint8_t count, all_ports_up, print_flag = 0;
3002 struct rte_eth_link link;
3005 printf("Checking link statuses...\n");
3007 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3009 RTE_ETH_FOREACH_DEV(portid) {
3010 if ((port_mask & (1 << portid)) == 0)
3012 memset(&link, 0, sizeof(link));
3013 ret = rte_eth_link_get_nowait(portid, &link);
3016 if (print_flag == 1)
3017 printf("Port %u link get failed: %s\n",
3018 portid, rte_strerror(-ret));
3021 /* print link status if flag set */
3022 if (print_flag == 1) {
3023 if (link.link_status)
3025 "Port%d Link Up. speed %u Mbps- %s\n",
3026 portid, link.link_speed,
3027 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3028 ("full-duplex") : ("half-duplex"));
3030 printf("Port %d Link Down\n", portid);
3033 /* clear all_ports_up flag if any link down */
3034 if (link.link_status == ETH_LINK_DOWN) {
3039 /* after finally printing all link status, get out */
3040 if (print_flag == 1)
3043 if (all_ports_up == 0) {
3045 rte_delay_ms(CHECK_INTERVAL);
3048 /* set the print_flag if all ports up or timeout */
3049 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3059 * This callback is for remove a port for a device. It has limitation because
3060 * it is not for multiple port removal for a device.
3061 * TODO: the device detach invoke will plan to be removed from user side to
3062 * eal. And convert all PMDs to free port resources on ether device closing.
3065 rmv_port_callback(void *arg)
3067 int need_to_start = 0;
3068 int org_no_link_check = no_link_check;
3069 portid_t port_id = (intptr_t)arg;
3070 struct rte_device *dev;
3072 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3074 if (!test_done && port_is_forwarding(port_id)) {
3076 stop_packet_forwarding();
3080 no_link_check = org_no_link_check;
3082 /* Save rte_device pointer before closing ethdev port */
3083 dev = rte_eth_devices[port_id].device;
3084 close_port(port_id);
3085 detach_device(dev); /* might be already removed or have more ports */
3088 start_packet_forwarding(0);
3091 /* This function is used by the interrupt thread */
3093 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3096 RTE_SET_USED(param);
3097 RTE_SET_USED(ret_param);
3099 if (type >= RTE_ETH_EVENT_MAX) {
3100 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3101 port_id, __func__, type);
3103 } else if (event_print_mask & (UINT32_C(1) << type)) {
3104 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3105 eth_event_desc[type]);
3110 case RTE_ETH_EVENT_NEW:
3111 ports[port_id].need_setup = 1;
3112 ports[port_id].port_status = RTE_PORT_HANDLING;
3114 case RTE_ETH_EVENT_INTR_RMV:
3115 if (port_id_is_invalid(port_id, DISABLED_WARN))
3117 if (rte_eal_alarm_set(100000,
3118 rmv_port_callback, (void *)(intptr_t)port_id))
3119 fprintf(stderr, "Could not set up deferred device removal\n");
3128 register_eth_event_callback(void)
3131 enum rte_eth_event_type event;
3133 for (event = RTE_ETH_EVENT_UNKNOWN;
3134 event < RTE_ETH_EVENT_MAX; event++) {
3135 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3140 TESTPMD_LOG(ERR, "Failed to register callback for "
3141 "%s event\n", eth_event_desc[event]);
3149 /* This function is used by the interrupt thread */
3151 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3152 __rte_unused void *arg)
3157 if (type >= RTE_DEV_EVENT_MAX) {
3158 fprintf(stderr, "%s called upon invalid event %d\n",
3164 case RTE_DEV_EVENT_REMOVE:
3165 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3167 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3169 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3174 * Because the user's callback is invoked in eal interrupt
3175 * callback, the interrupt callback need to be finished before
3176 * it can be unregistered when detaching device. So finish
3177 * callback soon and use a deferred removal to detach device
3178 * is need. It is a workaround, once the device detaching be
3179 * moved into the eal in the future, the deferred removal could
3182 if (rte_eal_alarm_set(100000,
3183 rmv_port_callback, (void *)(intptr_t)port_id))
3185 "Could not set up deferred device removal\n");
3187 case RTE_DEV_EVENT_ADD:
3188 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3190 /* TODO: After finish kernel driver binding,
3191 * begin to attach port.
3200 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3204 uint8_t mapping_found = 0;
3206 for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3207 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3208 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3209 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3210 tx_queue_stats_mappings[i].queue_id,
3211 tx_queue_stats_mappings[i].stats_counter_id);
3218 port->tx_queue_stats_mapping_enabled = 1;
3223 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3227 uint8_t mapping_found = 0;
3229 for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3230 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3231 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3232 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3233 rx_queue_stats_mappings[i].queue_id,
3234 rx_queue_stats_mappings[i].stats_counter_id);
3241 port->rx_queue_stats_mapping_enabled = 1;
3246 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3250 diag = set_tx_queue_stats_mapping_registers(pi, port);
3252 if (diag == -ENOTSUP) {
3253 port->tx_queue_stats_mapping_enabled = 0;
3254 printf("TX queue stats mapping not supported port id=%d\n", pi);
3257 rte_exit(EXIT_FAILURE,
3258 "set_tx_queue_stats_mapping_registers "
3259 "failed for port id=%d diag=%d\n",
3263 diag = set_rx_queue_stats_mapping_registers(pi, port);
3265 if (diag == -ENOTSUP) {
3266 port->rx_queue_stats_mapping_enabled = 0;
3267 printf("RX queue stats mapping not supported port id=%d\n", pi);
3270 rte_exit(EXIT_FAILURE,
3271 "set_rx_queue_stats_mapping_registers "
3272 "failed for port id=%d diag=%d\n",
3278 rxtx_port_config(struct rte_port *port)
3283 for (qid = 0; qid < nb_rxq; qid++) {
3284 offloads = port->rx_conf[qid].offloads;
3285 port->rx_conf[qid] = port->dev_info.default_rxconf;
3287 port->rx_conf[qid].offloads = offloads;
3289 /* Check if any Rx parameters have been passed */
3290 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3291 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3293 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3294 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3296 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3297 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3299 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3300 port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3302 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3303 port->rx_conf[qid].rx_drop_en = rx_drop_en;
3305 port->nb_rx_desc[qid] = nb_rxd;
3308 for (qid = 0; qid < nb_txq; qid++) {
3309 offloads = port->tx_conf[qid].offloads;
3310 port->tx_conf[qid] = port->dev_info.default_txconf;
3312 port->tx_conf[qid].offloads = offloads;
3314 /* Check if any Tx parameters have been passed */
3315 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3316 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3318 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3319 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3321 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3322 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3324 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3325 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3327 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3328 port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3330 port->nb_tx_desc[qid] = nb_txd;
3335 init_port_config(void)
3338 struct rte_port *port;
3341 RTE_ETH_FOREACH_DEV(pid) {
3343 port->dev_conf.fdir_conf = fdir_conf;
3345 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3350 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3351 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3352 rss_hf & port->dev_info.flow_type_rss_offloads;
3354 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3355 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3358 if (port->dcb_flag == 0) {
3359 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3360 port->dev_conf.rxmode.mq_mode =
3361 (enum rte_eth_rx_mq_mode)
3362 (rx_mq_mode & ETH_MQ_RX_RSS);
3364 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3367 rxtx_port_config(port);
3369 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3373 map_port_queue_stats_mapping_registers(pid, port);
3374 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3375 rte_pmd_ixgbe_bypass_init(pid);
3378 if (lsc_interrupt &&
3379 (rte_eth_devices[pid].data->dev_flags &
3380 RTE_ETH_DEV_INTR_LSC))
3381 port->dev_conf.intr_conf.lsc = 1;
3382 if (rmv_interrupt &&
3383 (rte_eth_devices[pid].data->dev_flags &
3384 RTE_ETH_DEV_INTR_RMV))
3385 port->dev_conf.intr_conf.rmv = 1;
3389 void set_port_slave_flag(portid_t slave_pid)
3391 struct rte_port *port;
3393 port = &ports[slave_pid];
3394 port->slave_flag = 1;
3397 void clear_port_slave_flag(portid_t slave_pid)
3399 struct rte_port *port;
3401 port = &ports[slave_pid];
3402 port->slave_flag = 0;
3405 uint8_t port_is_bonding_slave(portid_t slave_pid)
3407 struct rte_port *port;
3409 port = &ports[slave_pid];
3410 if ((rte_eth_devices[slave_pid].data->dev_flags &
3411 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3416 const uint16_t vlan_tags[] = {
3417 0, 1, 2, 3, 4, 5, 6, 7,
3418 8, 9, 10, 11, 12, 13, 14, 15,
3419 16, 17, 18, 19, 20, 21, 22, 23,
3420 24, 25, 26, 27, 28, 29, 30, 31
3424 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3425 enum dcb_mode_enable dcb_mode,
3426 enum rte_eth_nb_tcs num_tcs,
3431 struct rte_eth_rss_conf rss_conf;
3434 * Builds up the correct configuration for dcb+vt based on the vlan tags array
3435 * given above, and the number of traffic classes available for use.
3437 if (dcb_mode == DCB_VT_ENABLED) {
3438 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3439 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3440 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3441 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3443 /* VMDQ+DCB RX and TX configurations */
3444 vmdq_rx_conf->enable_default_pool = 0;
3445 vmdq_rx_conf->default_pool = 0;
3446 vmdq_rx_conf->nb_queue_pools =
3447 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3448 vmdq_tx_conf->nb_queue_pools =
3449 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3451 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3452 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3453 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3454 vmdq_rx_conf->pool_map[i].pools =
3455 1 << (i % vmdq_rx_conf->nb_queue_pools);
3457 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3458 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3459 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3462 /* set DCB mode of RX and TX of multiple queues */
3463 eth_conf->rxmode.mq_mode =
3464 (enum rte_eth_rx_mq_mode)
3465 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3466 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3468 struct rte_eth_dcb_rx_conf *rx_conf =
3469 ð_conf->rx_adv_conf.dcb_rx_conf;
3470 struct rte_eth_dcb_tx_conf *tx_conf =
3471 ð_conf->tx_adv_conf.dcb_tx_conf;
3473 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3475 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3479 rx_conf->nb_tcs = num_tcs;
3480 tx_conf->nb_tcs = num_tcs;
3482 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3483 rx_conf->dcb_tc[i] = i % num_tcs;
3484 tx_conf->dcb_tc[i] = i % num_tcs;
3487 eth_conf->rxmode.mq_mode =
3488 (enum rte_eth_rx_mq_mode)
3489 (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3490 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3491 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3495 eth_conf->dcb_capability_en =
3496 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3498 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3504 init_port_dcb_config(portid_t pid,
3505 enum dcb_mode_enable dcb_mode,
3506 enum rte_eth_nb_tcs num_tcs,
3509 struct rte_eth_conf port_conf;
3510 struct rte_port *rte_port;
3514 rte_port = &ports[pid];
3516 memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3517 /* Enter DCB configuration status */
3520 port_conf.rxmode = rte_port->dev_conf.rxmode;
3521 port_conf.txmode = rte_port->dev_conf.txmode;
3523 /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3524 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3527 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3529 /* re-configure the device . */
3530 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3534 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3538 /* If dev_info.vmdq_pool_base is greater than 0,
3539 * the queue id of vmdq pools is started after pf queues.
3541 if (dcb_mode == DCB_VT_ENABLED &&
3542 rte_port->dev_info.vmdq_pool_base > 0) {
3543 printf("VMDQ_DCB multi-queue mode is nonsensical"
3544 " for port %d.", pid);
3548 /* Assume the ports in testpmd have the same dcb capability
3549 * and has the same number of rxq and txq in dcb mode
3551 if (dcb_mode == DCB_VT_ENABLED) {
3552 if (rte_port->dev_info.max_vfs > 0) {
3553 nb_rxq = rte_port->dev_info.nb_rx_queues;
3554 nb_txq = rte_port->dev_info.nb_tx_queues;
3556 nb_rxq = rte_port->dev_info.max_rx_queues;
3557 nb_txq = rte_port->dev_info.max_tx_queues;
3560 /*if vt is disabled, use all pf queues */
3561 if (rte_port->dev_info.vmdq_pool_base == 0) {
3562 nb_rxq = rte_port->dev_info.max_rx_queues;
3563 nb_txq = rte_port->dev_info.max_tx_queues;
3565 nb_rxq = (queueid_t)num_tcs;
3566 nb_txq = (queueid_t)num_tcs;
3570 rx_free_thresh = 64;
3572 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3574 rxtx_port_config(rte_port);
3576 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3577 for (i = 0; i < RTE_DIM(vlan_tags); i++)
3578 rx_vft_set(pid, vlan_tags[i], 1);
3580 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3584 map_port_queue_stats_mapping_registers(pid, rte_port);
3586 rte_port->dcb_flag = 1;
3594 /* Configuration of Ethernet ports. */
3595 ports = rte_zmalloc("testpmd: ports",
3596 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3597 RTE_CACHE_LINE_SIZE);
3598 if (ports == NULL) {
3599 rte_exit(EXIT_FAILURE,
3600 "rte_zmalloc(%d struct rte_port) failed\n",
3604 /* Initialize ports NUMA structures */
3605 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3606 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3607 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3621 const char clr[] = { 27, '[', '2', 'J', '\0' };
3622 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3624 /* Clear screen and move to top left */
3625 printf("%s%s", clr, top_left);
3627 printf("\nPort statistics ====================================");
3628 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3629 nic_stats_display(fwd_ports_ids[i]);
3635 signal_handler(int signum)
3637 if (signum == SIGINT || signum == SIGTERM) {
3638 printf("\nSignal %d received, preparing to exit...\n",
3640 #ifdef RTE_LIBRTE_PDUMP
3641 /* uninitialize packet capture framework */
3644 #ifdef RTE_LIBRTE_LATENCY_STATS
3645 if (latencystats_enabled != 0)
3646 rte_latencystats_uninit();
3649 /* Set flag to indicate the force termination. */
3651 /* exit with the expected status */
3652 signal(signum, SIG_DFL);
3653 kill(getpid(), signum);
3658 main(int argc, char** argv)
3665 signal(SIGINT, signal_handler);
3666 signal(SIGTERM, signal_handler);
3668 testpmd_logtype = rte_log_register("testpmd");
3669 if (testpmd_logtype < 0)
3670 rte_exit(EXIT_FAILURE, "Cannot register log type");
3671 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3673 diag = rte_eal_init(argc, argv);
3675 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3676 rte_strerror(rte_errno));
3678 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3679 rte_exit(EXIT_FAILURE,
3680 "Secondary process type not supported.\n");
3682 ret = register_eth_event_callback();
3684 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3686 #ifdef RTE_LIBRTE_PDUMP
3687 /* initialize packet capture framework */
3692 RTE_ETH_FOREACH_DEV(port_id) {
3693 ports_ids[count] = port_id;
3696 nb_ports = (portid_t) count;
3698 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3700 /* allocate port structures, and init them */
3703 set_def_fwd_config();
3705 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3706 "Check the core mask argument\n");
3708 /* Bitrate/latency stats disabled by default */
3709 #ifdef RTE_LIBRTE_BITRATE
3710 bitrate_enabled = 0;
3712 #ifdef RTE_LIBRTE_LATENCY_STATS
3713 latencystats_enabled = 0;
3716 /* on FreeBSD, mlockall() is disabled by default */
3717 #ifdef RTE_EXEC_ENV_FREEBSD
3726 launch_args_parse(argc, argv);
3728 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3729 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3733 if (tx_first && interactive)
3734 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3735 "interactive mode.\n");
3737 if (tx_first && lsc_interrupt) {
3738 printf("Warning: lsc_interrupt needs to be off when "
3739 " using tx_first. Disabling.\n");
3743 if (!nb_rxq && !nb_txq)
3744 printf("Warning: Either rx or tx queues should be non-zero\n");
3746 if (nb_rxq > 1 && nb_rxq > nb_txq)
3747 printf("Warning: nb_rxq=%d enables RSS configuration, "
3748 "but nb_txq=%d will prevent to fully test it.\n",
3754 ret = rte_dev_hotplug_handle_enable();
3757 "fail to enable hotplug handling.");
3761 ret = rte_dev_event_monitor_start();
3764 "fail to start device event monitoring.");
3768 ret = rte_dev_event_callback_register(NULL,
3769 dev_event_callback, NULL);
3772 "fail to register device event callback\n");
3777 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3778 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3780 /* set all ports to promiscuous mode by default */
3781 RTE_ETH_FOREACH_DEV(port_id) {
3782 ret = rte_eth_promiscuous_enable(port_id);
3784 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3785 port_id, rte_strerror(-ret));
3788 /* Init metrics library */
3789 rte_metrics_init(rte_socket_id());
3791 #ifdef RTE_LIBRTE_LATENCY_STATS
3792 if (latencystats_enabled != 0) {
3793 int ret = rte_latencystats_init(1, NULL);
3795 printf("Warning: latencystats init()"
3796 " returned error %d\n", ret);
3797 printf("Latencystats running on lcore %d\n",
3798 latencystats_lcore_id);
3802 /* Setup bitrate stats */
3803 #ifdef RTE_LIBRTE_BITRATE
3804 if (bitrate_enabled != 0) {
3805 bitrate_data = rte_stats_bitrate_create();
3806 if (bitrate_data == NULL)
3807 rte_exit(EXIT_FAILURE,
3808 "Could not allocate bitrate data.\n");
3809 rte_stats_bitrate_reg(bitrate_data);
3813 #ifdef RTE_LIBRTE_CMDLINE
3814 if (strlen(cmdline_filename) != 0)
3815 cmdline_read_from_file(cmdline_filename);
3817 if (interactive == 1) {
3819 printf("Start automatic packet forwarding\n");
3820 start_packet_forwarding(0);
3832 printf("No commandline core given, start packet forwarding\n");
3833 start_packet_forwarding(tx_first);
3834 if (stats_period != 0) {
3835 uint64_t prev_time = 0, cur_time, diff_time = 0;
3836 uint64_t timer_period;
3838 /* Convert to number of cycles */
3839 timer_period = stats_period * rte_get_timer_hz();
3841 while (f_quit == 0) {
3842 cur_time = rte_get_timer_cycles();
3843 diff_time += cur_time - prev_time;
3845 if (diff_time >= timer_period) {
3847 /* Reset the timer */
3850 /* Sleep to avoid unnecessary checks */
3851 prev_time = cur_time;
3856 printf("Press enter to exit\n");
3857 rc = read(0, &c, 1);
3863 ret = rte_eal_cleanup();
3865 rte_exit(EXIT_FAILURE,
3866 "EAL cleanup failed: %s\n", strerror(-ret));
3868 return EXIT_SUCCESS;