1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
13 #include <sys/types.h>
17 #include <sys/queue.h>
24 #include <rte_common.h>
25 #include <rte_errno.h>
26 #include <rte_byteorder.h>
28 #include <rte_debug.h>
29 #include <rte_cycles.h>
30 #include <rte_memory.h>
31 #include <rte_memcpy.h>
32 #include <rte_launch.h>
34 #include <rte_alarm.h>
35 #include <rte_per_lcore.h>
36 #include <rte_lcore.h>
37 #include <rte_atomic.h>
38 #include <rte_branch_prediction.h>
39 #include <rte_mempool.h>
40 #include <rte_malloc.h>
42 #include <rte_mbuf_pool_ops.h>
43 #include <rte_interrupts.h>
45 #include <rte_ether.h>
46 #include <rte_ethdev.h>
48 #include <rte_string_fns.h>
49 #ifdef RTE_LIBRTE_IXGBE_PMD
50 #include <rte_pmd_ixgbe.h>
52 #ifdef RTE_LIBRTE_PDUMP
53 #include <rte_pdump.h>
56 #include <rte_metrics.h>
57 #ifdef RTE_LIBRTE_BITRATE
58 #include <rte_bitrate.h>
60 #ifdef RTE_LIBRTE_LATENCY_STATS
61 #include <rte_latencystats.h>
67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
68 #define HUGE_FLAG (0x40000)
70 #define HUGE_FLAG MAP_HUGETLB
73 #ifndef MAP_HUGE_SHIFT
74 /* older kernels (or FreeBSD) will not have this define */
75 #define HUGE_SHIFT (26)
77 #define HUGE_SHIFT MAP_HUGE_SHIFT
80 #define EXTMEM_HEAP_NAME "extmem"
81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
83 uint16_t verbose_level = 0; /**< Silent by default. */
84 int testpmd_logtype; /**< Log type for testpmd logs */
86 /* use master core for command line ? */
87 uint8_t interactive = 0;
88 uint8_t auto_start = 0;
90 char cmdline_filename[PATH_MAX] = {0};
93 * NUMA support configuration.
94 * When set, the NUMA support attempts to dispatch the allocation of the
95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
96 * probed ports among the CPU sockets 0 and 1.
97 * Otherwise, all memory is allocated from CPU socket 0.
99 uint8_t numa_support = 1; /**< numa enabled by default */
102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is
105 uint8_t socket_num = UMA_NO_CONFIG;
108 * Select mempool allocation type:
109 * - native: use regular DPDK memory
110 * - anon: use regular DPDK memory to create mempool, but populate using
111 * anonymous memory (may not be IOVA-contiguous)
112 * - xmem: use externally allocated hugepage memory
114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
117 * Store specified sockets on which memory pool to be used by ports
120 uint8_t port_numa[RTE_MAX_ETHPORTS];
123 * Store specified sockets on which RX ring to be used by ports
126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
129 * Store specified sockets on which TX ring to be used by ports
132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
135 * Record the Ethernet address of peer target ports to which packets are
137 * Must be instantiated with the ethernet addresses of peer traffic generator
140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
141 portid_t nb_peer_eth_addrs = 0;
144 * Probed Target Environment.
146 struct rte_port *ports; /**< For all probed ethernet ports. */
147 portid_t nb_ports; /**< Number of probed ethernet ports. */
148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */
151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
154 * Test Forwarding Configuration.
155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports
158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
160 portid_t nb_cfg_ports; /**< Number of configured ports. */
161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */
163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */
166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */
170 * Forwarding engines.
172 struct fwd_engine * fwd_engines[] = {
182 #ifdef RTE_LIBRTE_IEEE1588
183 &ieee1588_fwd_engine,
188 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
189 uint16_t mempool_flags;
191 struct fwd_config cur_fwd_config;
192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
193 uint32_t retry_enabled;
194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
198 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if
199 * specified on command-line. */
200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
203 * In container, it cannot terminate the process which running with 'stats-period'
204 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
209 * Configuration of packet segments used by the "txonly" processing engine.
211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
213 TXONLY_DEF_PACKET_LEN,
215 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
218 /**< Split policy for packets to TX. */
220 uint8_t txonly_multi_flow;
221 /**< Whether multiple flows are generated in TXONLY mode. */
223 uint32_t tx_pkt_times_inter;
224 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
226 uint32_t tx_pkt_times_intra;
227 /**< Timings for send scheduling in TXONLY mode, time between packets. */
229 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
230 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
232 /* current configuration is in DCB or not,0 means it is not in DCB mode */
233 uint8_t dcb_config = 0;
235 /* Whether the dcb is in testing status */
236 uint8_t dcb_test = 0;
239 * Configurable number of RX/TX queues.
241 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
242 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
243 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
246 * Configurable number of RX/TX ring descriptors.
247 * Defaults are supplied by drivers via ethdev.
249 #define RTE_TEST_RX_DESC_DEFAULT 0
250 #define RTE_TEST_TX_DESC_DEFAULT 0
251 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
252 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
254 #define RTE_PMD_PARAM_UNSET -1
256 * Configurable values of RX and TX ring threshold registers.
259 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
260 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
261 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
263 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
264 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
265 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
268 * Configurable value of RX free threshold.
270 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
273 * Configurable value of RX drop enable.
275 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
278 * Configurable value of TX free threshold.
280 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
283 * Configurable value of TX RS bit threshold.
285 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
288 * Configurable value of buffered packets before sending.
290 uint16_t noisy_tx_sw_bufsz;
293 * Configurable value of packet buffer timeout.
295 uint16_t noisy_tx_sw_buf_flush_time;
298 * Configurable value for size of VNF internal memory area
299 * used for simulating noisy neighbour behaviour
301 uint64_t noisy_lkup_mem_sz;
304 * Configurable value of number of random writes done in
305 * VNF simulation memory area.
307 uint64_t noisy_lkup_num_writes;
310 * Configurable value of number of random reads done in
311 * VNF simulation memory area.
313 uint64_t noisy_lkup_num_reads;
316 * Configurable value of number of random reads/writes done in
317 * VNF simulation memory area.
319 uint64_t noisy_lkup_num_reads_writes;
322 * Receive Side Scaling (RSS) configuration.
324 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
327 * Port topology configuration
329 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
332 * Avoids to flush all the RX streams before starts forwarding.
334 uint8_t no_flush_rx = 0; /* flush by default */
337 * Flow API isolated mode.
339 uint8_t flow_isolate_all;
342 * Avoids to check link status when starting/stopping a port.
344 uint8_t no_link_check = 0; /* check by default */
347 * Don't automatically start all ports in interactive mode.
349 uint8_t no_device_start = 0;
352 * Enable link status change notification
354 uint8_t lsc_interrupt = 1; /* enabled by default */
357 * Enable device removal notification.
359 uint8_t rmv_interrupt = 1; /* enabled by default */
361 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
363 /* After attach, port setup is called on event or by iterator */
364 bool setup_on_probe_event = true;
366 /* Clear ptypes on port initialization. */
367 uint8_t clear_ptypes = true;
369 /* Pretty printing of ethdev events */
370 static const char * const eth_event_desc[] = {
371 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
372 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
373 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
374 [RTE_ETH_EVENT_INTR_RESET] = "reset",
375 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
376 [RTE_ETH_EVENT_IPSEC] = "IPsec",
377 [RTE_ETH_EVENT_MACSEC] = "MACsec",
378 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
379 [RTE_ETH_EVENT_NEW] = "device probed",
380 [RTE_ETH_EVENT_DESTROY] = "device released",
381 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
382 [RTE_ETH_EVENT_MAX] = NULL,
386 * Display or mask ether events
387 * Default to all events except VF_MBOX
389 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
390 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
391 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
392 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
393 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
394 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
395 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
396 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
398 * Decide if all memory are locked for performance.
403 * NIC bypass mode configuration options.
406 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
407 /* The NIC bypass watchdog timeout. */
408 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
412 #ifdef RTE_LIBRTE_LATENCY_STATS
415 * Set when latency stats is enabled in the commandline
417 uint8_t latencystats_enabled;
420 * Lcore ID to serive latency statistics.
422 lcoreid_t latencystats_lcore_id = -1;
427 * Ethernet device configuration.
429 struct rte_eth_rxmode rx_mode = {
430 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
431 /**< Default maximum frame length. */
434 struct rte_eth_txmode tx_mode = {
435 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
438 struct rte_fdir_conf fdir_conf = {
439 .mode = RTE_FDIR_MODE_NONE,
440 .pballoc = RTE_FDIR_PBALLOC_64K,
441 .status = RTE_FDIR_REPORT_STATUS,
443 .vlan_tci_mask = 0xFFEF,
445 .src_ip = 0xFFFFFFFF,
446 .dst_ip = 0xFFFFFFFF,
449 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
450 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
452 .src_port_mask = 0xFFFF,
453 .dst_port_mask = 0xFFFF,
454 .mac_addr_byte_mask = 0xFF,
455 .tunnel_type_mask = 1,
456 .tunnel_id_mask = 0xFFFFFFFF,
461 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
463 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
464 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
466 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
467 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
469 uint16_t nb_tx_queue_stats_mappings = 0;
470 uint16_t nb_rx_queue_stats_mappings = 0;
473 * Display zero values by default for xstats
475 uint8_t xstats_hide_zero;
477 unsigned int num_sockets = 0;
478 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
480 #ifdef RTE_LIBRTE_BITRATE
481 /* Bitrate statistics */
482 struct rte_stats_bitrates *bitrate_data;
483 lcoreid_t bitrate_lcore_id;
484 uint8_t bitrate_enabled;
487 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
488 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
491 * hexadecimal bitmask of RX mq mode can be enabled.
493 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
495 /* Forward function declarations */
496 static void setup_attached_port(portid_t pi);
497 static void map_port_queue_stats_mapping_registers(portid_t pi,
498 struct rte_port *port);
499 static void check_all_ports_link_status(uint32_t port_mask);
500 static int eth_event_callback(portid_t port_id,
501 enum rte_eth_event_type type,
502 void *param, void *ret_param);
503 static void dev_event_callback(const char *device_name,
504 enum rte_dev_event_type type,
508 * Check if all the ports are started.
509 * If yes, return positive value. If not, return zero.
511 static int all_ports_started(void);
513 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
514 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
516 /* Holds the registered mbuf dynamic flags names. */
517 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
520 * Helper function to check if socket is already discovered.
521 * If yes, return positive value. If not, return zero.
524 new_socket_id(unsigned int socket_id)
528 for (i = 0; i < num_sockets; i++) {
529 if (socket_ids[i] == socket_id)
536 * Setup default configuration.
539 set_default_fwd_lcores_config(void)
543 unsigned int sock_num;
546 for (i = 0; i < RTE_MAX_LCORE; i++) {
547 if (!rte_lcore_is_enabled(i))
549 sock_num = rte_lcore_to_socket_id(i);
550 if (new_socket_id(sock_num)) {
551 if (num_sockets >= RTE_MAX_NUMA_NODES) {
552 rte_exit(EXIT_FAILURE,
553 "Total sockets greater than %u\n",
556 socket_ids[num_sockets++] = sock_num;
558 if (i == rte_get_master_lcore())
560 fwd_lcores_cpuids[nb_lc++] = i;
562 nb_lcores = (lcoreid_t) nb_lc;
563 nb_cfg_lcores = nb_lcores;
568 set_def_peer_eth_addrs(void)
572 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
573 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
574 peer_eth_addrs[i].addr_bytes[5] = i;
579 set_default_fwd_ports_config(void)
584 RTE_ETH_FOREACH_DEV(pt_id) {
585 fwd_ports_ids[i++] = pt_id;
587 /* Update sockets info according to the attached device */
588 int socket_id = rte_eth_dev_socket_id(pt_id);
589 if (socket_id >= 0 && new_socket_id(socket_id)) {
590 if (num_sockets >= RTE_MAX_NUMA_NODES) {
591 rte_exit(EXIT_FAILURE,
592 "Total sockets greater than %u\n",
595 socket_ids[num_sockets++] = socket_id;
599 nb_cfg_ports = nb_ports;
600 nb_fwd_ports = nb_ports;
604 set_def_fwd_config(void)
606 set_default_fwd_lcores_config();
607 set_def_peer_eth_addrs();
608 set_default_fwd_ports_config();
611 /* extremely pessimistic estimation of memory required to create a mempool */
613 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
615 unsigned int n_pages, mbuf_per_pg, leftover;
616 uint64_t total_mem, mbuf_mem, obj_sz;
618 /* there is no good way to predict how much space the mempool will
619 * occupy because it will allocate chunks on the fly, and some of those
620 * will come from default DPDK memory while some will come from our
621 * external memory, so just assume 128MB will be enough for everyone.
623 uint64_t hdr_mem = 128 << 20;
625 /* account for possible non-contiguousness */
626 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
628 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
632 mbuf_per_pg = pgsz / obj_sz;
633 leftover = (nb_mbufs % mbuf_per_pg) > 0;
634 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
636 mbuf_mem = n_pages * pgsz;
638 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
640 if (total_mem > SIZE_MAX) {
641 TESTPMD_LOG(ERR, "Memory size too big\n");
644 *out = (size_t)total_mem;
650 pagesz_flags(uint64_t page_sz)
652 /* as per mmap() manpage, all page sizes are log2 of page size
653 * shifted by MAP_HUGE_SHIFT
655 int log2 = rte_log2_u64(page_sz);
657 return (log2 << HUGE_SHIFT);
661 alloc_mem(size_t memsz, size_t pgsz, bool huge)
666 /* allocate anonymous hugepages */
667 flags = MAP_ANONYMOUS | MAP_PRIVATE;
669 flags |= HUGE_FLAG | pagesz_flags(pgsz);
671 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
672 if (addr == MAP_FAILED)
678 struct extmem_param {
682 rte_iova_t *iova_table;
683 unsigned int iova_table_len;
687 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
690 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
691 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */
692 unsigned int cur_page, n_pages, pgsz_idx;
693 size_t mem_sz, cur_pgsz;
694 rte_iova_t *iovas = NULL;
698 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
699 /* skip anything that is too big */
700 if (pgsizes[pgsz_idx] > SIZE_MAX)
703 cur_pgsz = pgsizes[pgsz_idx];
705 /* if we were told not to allocate hugepages, override */
707 cur_pgsz = sysconf(_SC_PAGESIZE);
709 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
711 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
715 /* allocate our memory */
716 addr = alloc_mem(mem_sz, cur_pgsz, huge);
718 /* if we couldn't allocate memory with a specified page size,
719 * that doesn't mean we can't do it with other page sizes, so
725 /* store IOVA addresses for every page in this memory area */
726 n_pages = mem_sz / cur_pgsz;
728 iovas = malloc(sizeof(*iovas) * n_pages);
731 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
734 /* lock memory if it's not huge pages */
738 /* populate IOVA addresses */
739 for (cur_page = 0; cur_page < n_pages; cur_page++) {
744 offset = cur_pgsz * cur_page;
745 cur = RTE_PTR_ADD(addr, offset);
747 /* touch the page before getting its IOVA */
748 *(volatile char *)cur = 0;
750 iova = rte_mem_virt2iova(cur);
752 iovas[cur_page] = iova;
757 /* if we couldn't allocate anything */
763 param->pgsz = cur_pgsz;
764 param->iova_table = iovas;
765 param->iova_table_len = n_pages;
772 munmap(addr, mem_sz);
778 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
780 struct extmem_param param;
783 memset(¶m, 0, sizeof(param));
785 /* check if our heap exists */
786 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
788 /* create our heap */
789 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
791 TESTPMD_LOG(ERR, "Cannot create heap\n");
796 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
798 TESTPMD_LOG(ERR, "Cannot create memory area\n");
802 /* we now have a valid memory area, so add it to heap */
803 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
804 param.addr, param.len, param.iova_table,
805 param.iova_table_len, param.pgsz);
807 /* when using VFIO, memory is automatically mapped for DMA by EAL */
809 /* not needed any more */
810 free(param.iova_table);
813 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
814 munmap(param.addr, param.len);
820 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
826 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
827 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
832 RTE_ETH_FOREACH_DEV(pid) {
833 struct rte_eth_dev *dev =
834 &rte_eth_devices[pid];
836 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
840 "unable to DMA unmap addr 0x%p "
842 memhdr->addr, dev->data->name);
845 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
848 "unable to un-register addr 0x%p\n", memhdr->addr);
853 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
854 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
857 size_t page_size = sysconf(_SC_PAGESIZE);
860 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
864 "unable to register addr 0x%p\n", memhdr->addr);
867 RTE_ETH_FOREACH_DEV(pid) {
868 struct rte_eth_dev *dev =
869 &rte_eth_devices[pid];
871 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
875 "unable to DMA map addr 0x%p "
877 memhdr->addr, dev->data->name);
883 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
884 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
886 struct rte_pktmbuf_extmem *xmem;
887 unsigned int ext_num, zone_num, elt_num;
890 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
891 elt_num = EXTBUF_ZONE_SIZE / elt_size;
892 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
894 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
896 TESTPMD_LOG(ERR, "Cannot allocate memory for "
897 "external buffer descriptors\n");
901 for (ext_num = 0; ext_num < zone_num; ext_num++) {
902 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
903 const struct rte_memzone *mz;
904 char mz_name[RTE_MEMZONE_NAMESIZE];
907 ret = snprintf(mz_name, sizeof(mz_name),
908 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
909 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
910 errno = ENAMETOOLONG;
914 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
916 RTE_MEMZONE_IOVA_CONTIG |
918 RTE_MEMZONE_SIZE_HINT_ONLY,
922 * The caller exits on external buffer creation
923 * error, so there is no need to free memzones.
929 xseg->buf_ptr = mz->addr;
930 xseg->buf_iova = mz->iova;
931 xseg->buf_len = EXTBUF_ZONE_SIZE;
932 xseg->elt_size = elt_size;
934 if (ext_num == 0 && xmem != NULL) {
943 * Configuration initialisation done once at init time.
945 static struct rte_mempool *
946 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
947 unsigned int socket_id)
949 char pool_name[RTE_MEMPOOL_NAMESIZE];
950 struct rte_mempool *rte_mp = NULL;
953 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
954 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
957 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
958 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
960 switch (mp_alloc_type) {
961 case MP_ALLOC_NATIVE:
963 /* wrapper to rte_mempool_create() */
964 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
965 rte_mbuf_best_mempool_ops());
966 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
967 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
972 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
973 mb_size, (unsigned int) mb_mempool_cache,
974 sizeof(struct rte_pktmbuf_pool_private),
975 socket_id, mempool_flags);
979 if (rte_mempool_populate_anon(rte_mp) == 0) {
980 rte_mempool_free(rte_mp);
984 rte_pktmbuf_pool_init(rte_mp, NULL);
985 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
986 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
990 case MP_ALLOC_XMEM_HUGE:
993 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
995 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
996 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
999 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1000 if (heap_socket < 0)
1001 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1003 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1004 rte_mbuf_best_mempool_ops());
1005 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1006 mb_mempool_cache, 0, mbuf_seg_size,
1012 struct rte_pktmbuf_extmem *ext_mem;
1013 unsigned int ext_num;
1015 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1016 socket_id, pool_name, &ext_mem);
1018 rte_exit(EXIT_FAILURE,
1019 "Can't create pinned data buffers\n");
1021 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1022 rte_mbuf_best_mempool_ops());
1023 rte_mp = rte_pktmbuf_pool_create_extbuf
1024 (pool_name, nb_mbuf, mb_mempool_cache,
1025 0, mbuf_seg_size, socket_id,
1032 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1037 if (rte_mp == NULL) {
1038 rte_exit(EXIT_FAILURE,
1039 "Creation of mbuf pool for socket %u failed: %s\n",
1040 socket_id, rte_strerror(rte_errno));
1041 } else if (verbose_level > 0) {
1042 rte_mempool_dump(stdout, rte_mp);
1048 * Check given socket id is valid or not with NUMA mode,
1049 * if valid, return 0, else return -1
1052 check_socket_id(const unsigned int socket_id)
1054 static int warning_once = 0;
1056 if (new_socket_id(socket_id)) {
1057 if (!warning_once && numa_support)
1058 printf("Warning: NUMA should be configured manually by"
1059 " using --port-numa-config and"
1060 " --ring-numa-config parameters along with"
1069 * Get the allowed maximum number of RX queues.
1070 * *pid return the port id which has minimal value of
1071 * max_rx_queues in all ports.
1074 get_allowed_max_nb_rxq(portid_t *pid)
1076 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1077 bool max_rxq_valid = false;
1079 struct rte_eth_dev_info dev_info;
1081 RTE_ETH_FOREACH_DEV(pi) {
1082 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1085 max_rxq_valid = true;
1086 if (dev_info.max_rx_queues < allowed_max_rxq) {
1087 allowed_max_rxq = dev_info.max_rx_queues;
1091 return max_rxq_valid ? allowed_max_rxq : 0;
1095 * Check input rxq is valid or not.
1096 * If input rxq is not greater than any of maximum number
1097 * of RX queues of all ports, it is valid.
1098 * if valid, return 0, else return -1
1101 check_nb_rxq(queueid_t rxq)
1103 queueid_t allowed_max_rxq;
1106 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1107 if (rxq > allowed_max_rxq) {
1108 printf("Fail: input rxq (%u) can't be greater "
1109 "than max_rx_queues (%u) of port %u\n",
1119 * Get the allowed maximum number of TX queues.
1120 * *pid return the port id which has minimal value of
1121 * max_tx_queues in all ports.
1124 get_allowed_max_nb_txq(portid_t *pid)
1126 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1127 bool max_txq_valid = false;
1129 struct rte_eth_dev_info dev_info;
1131 RTE_ETH_FOREACH_DEV(pi) {
1132 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1135 max_txq_valid = true;
1136 if (dev_info.max_tx_queues < allowed_max_txq) {
1137 allowed_max_txq = dev_info.max_tx_queues;
1141 return max_txq_valid ? allowed_max_txq : 0;
1145 * Check input txq is valid or not.
1146 * If input txq is not greater than any of maximum number
1147 * of TX queues of all ports, it is valid.
1148 * if valid, return 0, else return -1
1151 check_nb_txq(queueid_t txq)
1153 queueid_t allowed_max_txq;
1156 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1157 if (txq > allowed_max_txq) {
1158 printf("Fail: input txq (%u) can't be greater "
1159 "than max_tx_queues (%u) of port %u\n",
1169 * Get the allowed maximum number of RXDs of every rx queue.
1170 * *pid return the port id which has minimal value of
1171 * max_rxd in all queues of all ports.
1174 get_allowed_max_nb_rxd(portid_t *pid)
1176 uint16_t allowed_max_rxd = UINT16_MAX;
1178 struct rte_eth_dev_info dev_info;
1180 RTE_ETH_FOREACH_DEV(pi) {
1181 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1184 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1185 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1189 return allowed_max_rxd;
1193 * Get the allowed minimal number of RXDs of every rx queue.
1194 * *pid return the port id which has minimal value of
1195 * min_rxd in all queues of all ports.
1198 get_allowed_min_nb_rxd(portid_t *pid)
1200 uint16_t allowed_min_rxd = 0;
1202 struct rte_eth_dev_info dev_info;
1204 RTE_ETH_FOREACH_DEV(pi) {
1205 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1208 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1209 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1214 return allowed_min_rxd;
1218 * Check input rxd is valid or not.
1219 * If input rxd is not greater than any of maximum number
1220 * of RXDs of every Rx queues and is not less than any of
1221 * minimal number of RXDs of every Rx queues, it is valid.
1222 * if valid, return 0, else return -1
1225 check_nb_rxd(queueid_t rxd)
1227 uint16_t allowed_max_rxd;
1228 uint16_t allowed_min_rxd;
1231 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1232 if (rxd > allowed_max_rxd) {
1233 printf("Fail: input rxd (%u) can't be greater "
1234 "than max_rxds (%u) of port %u\n",
1241 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1242 if (rxd < allowed_min_rxd) {
1243 printf("Fail: input rxd (%u) can't be less "
1244 "than min_rxds (%u) of port %u\n",
1255 * Get the allowed maximum number of TXDs of every rx queues.
1256 * *pid return the port id which has minimal value of
1257 * max_txd in every tx queue.
1260 get_allowed_max_nb_txd(portid_t *pid)
1262 uint16_t allowed_max_txd = UINT16_MAX;
1264 struct rte_eth_dev_info dev_info;
1266 RTE_ETH_FOREACH_DEV(pi) {
1267 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1270 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1271 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1275 return allowed_max_txd;
1279 * Get the allowed maximum number of TXDs of every tx queues.
1280 * *pid return the port id which has minimal value of
1281 * min_txd in every tx queue.
1284 get_allowed_min_nb_txd(portid_t *pid)
1286 uint16_t allowed_min_txd = 0;
1288 struct rte_eth_dev_info dev_info;
1290 RTE_ETH_FOREACH_DEV(pi) {
1291 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1294 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1295 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1300 return allowed_min_txd;
1304 * Check input txd is valid or not.
1305 * If input txd is not greater than any of maximum number
1306 * of TXDs of every Rx queues, it is valid.
1307 * if valid, return 0, else return -1
1310 check_nb_txd(queueid_t txd)
1312 uint16_t allowed_max_txd;
1313 uint16_t allowed_min_txd;
1316 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1317 if (txd > allowed_max_txd) {
1318 printf("Fail: input txd (%u) can't be greater "
1319 "than max_txds (%u) of port %u\n",
1326 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1327 if (txd < allowed_min_txd) {
1328 printf("Fail: input txd (%u) can't be less "
1329 "than min_txds (%u) of port %u\n",
1340 * Get the allowed maximum number of hairpin queues.
1341 * *pid return the port id which has minimal value of
1342 * max_hairpin_queues in all ports.
1345 get_allowed_max_nb_hairpinq(portid_t *pid)
1347 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1349 struct rte_eth_hairpin_cap cap;
1351 RTE_ETH_FOREACH_DEV(pi) {
1352 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1356 if (cap.max_nb_queues < allowed_max_hairpinq) {
1357 allowed_max_hairpinq = cap.max_nb_queues;
1361 return allowed_max_hairpinq;
1365 * Check input hairpin is valid or not.
1366 * If input hairpin is not greater than any of maximum number
1367 * of hairpin queues of all ports, it is valid.
1368 * if valid, return 0, else return -1
1371 check_nb_hairpinq(queueid_t hairpinq)
1373 queueid_t allowed_max_hairpinq;
1376 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1377 if (hairpinq > allowed_max_hairpinq) {
1378 printf("Fail: input hairpin (%u) can't be greater "
1379 "than max_hairpin_queues (%u) of port %u\n",
1380 hairpinq, allowed_max_hairpinq, pid);
1390 struct rte_port *port;
1391 struct rte_mempool *mbp;
1392 unsigned int nb_mbuf_per_pool;
1394 uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1395 struct rte_gro_param gro_param;
1402 memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1404 /* Configuration of logical cores. */
1405 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1406 sizeof(struct fwd_lcore *) * nb_lcores,
1407 RTE_CACHE_LINE_SIZE);
1408 if (fwd_lcores == NULL) {
1409 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1410 "failed\n", nb_lcores);
1412 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1413 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1414 sizeof(struct fwd_lcore),
1415 RTE_CACHE_LINE_SIZE);
1416 if (fwd_lcores[lc_id] == NULL) {
1417 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1420 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1423 RTE_ETH_FOREACH_DEV(pid) {
1425 /* Apply default TxRx configuration for all ports */
1426 port->dev_conf.txmode = tx_mode;
1427 port->dev_conf.rxmode = rx_mode;
1429 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1431 rte_exit(EXIT_FAILURE,
1432 "rte_eth_dev_info_get() failed\n");
1434 if (!(port->dev_info.tx_offload_capa &
1435 DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1436 port->dev_conf.txmode.offloads &=
1437 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1439 if (port_numa[pid] != NUMA_NO_CONFIG)
1440 port_per_socket[port_numa[pid]]++;
1442 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1445 * if socket_id is invalid,
1446 * set to the first available socket.
1448 if (check_socket_id(socket_id) < 0)
1449 socket_id = socket_ids[0];
1450 port_per_socket[socket_id]++;
1454 /* Apply Rx offloads configuration */
1455 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1456 port->rx_conf[k].offloads =
1457 port->dev_conf.rxmode.offloads;
1458 /* Apply Tx offloads configuration */
1459 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1460 port->tx_conf[k].offloads =
1461 port->dev_conf.txmode.offloads;
1463 /* set flag to initialize port/queue */
1464 port->need_reconfig = 1;
1465 port->need_reconfig_queues = 1;
1466 port->tx_metadata = 0;
1468 /* Check for maximum number of segments per MTU. Accordingly
1469 * update the mbuf data size.
1471 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1472 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1473 data_size = rx_mode.max_rx_pkt_len /
1474 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1476 if ((data_size + RTE_PKTMBUF_HEADROOM) >
1478 mbuf_data_size = data_size +
1479 RTE_PKTMBUF_HEADROOM;
1486 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1490 * Create pools of mbuf.
1491 * If NUMA support is disabled, create a single pool of mbuf in
1492 * socket 0 memory by default.
1493 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1495 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1496 * nb_txd can be configured at run time.
1498 if (param_total_num_mbufs)
1499 nb_mbuf_per_pool = param_total_num_mbufs;
1501 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1502 (nb_lcores * mb_mempool_cache) +
1503 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1504 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1510 for (i = 0; i < num_sockets; i++)
1511 mempools[i] = mbuf_pool_create(mbuf_data_size,
1515 if (socket_num == UMA_NO_CONFIG)
1516 mempools[0] = mbuf_pool_create(mbuf_data_size,
1517 nb_mbuf_per_pool, 0);
1519 mempools[socket_num] = mbuf_pool_create
1527 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1528 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1530 * Records which Mbuf pool to use by each logical core, if needed.
1532 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1533 mbp = mbuf_pool_find(
1534 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1537 mbp = mbuf_pool_find(0);
1538 fwd_lcores[lc_id]->mbp = mbp;
1539 /* initialize GSO context */
1540 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1541 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1542 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1543 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1545 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1548 /* Configuration of packet forwarding streams. */
1549 if (init_fwd_streams() < 0)
1550 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1554 /* create a gro context for each lcore */
1555 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1556 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1557 gro_param.max_item_per_flow = MAX_PKT_BURST;
1558 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1559 gro_param.socket_id = rte_lcore_to_socket_id(
1560 fwd_lcores_cpuids[lc_id]);
1561 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1562 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1563 rte_exit(EXIT_FAILURE,
1564 "rte_gro_ctx_create() failed\n");
1571 reconfig(portid_t new_port_id, unsigned socket_id)
1573 struct rte_port *port;
1576 /* Reconfiguration of Ethernet ports. */
1577 port = &ports[new_port_id];
1579 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1583 /* set flag to initialize port/queue */
1584 port->need_reconfig = 1;
1585 port->need_reconfig_queues = 1;
1586 port->socket_id = socket_id;
1593 init_fwd_streams(void)
1596 struct rte_port *port;
1597 streamid_t sm_id, nb_fwd_streams_new;
1600 /* set socket id according to numa or not */
1601 RTE_ETH_FOREACH_DEV(pid) {
1603 if (nb_rxq > port->dev_info.max_rx_queues) {
1604 printf("Fail: nb_rxq(%d) is greater than "
1605 "max_rx_queues(%d)\n", nb_rxq,
1606 port->dev_info.max_rx_queues);
1609 if (nb_txq > port->dev_info.max_tx_queues) {
1610 printf("Fail: nb_txq(%d) is greater than "
1611 "max_tx_queues(%d)\n", nb_txq,
1612 port->dev_info.max_tx_queues);
1616 if (port_numa[pid] != NUMA_NO_CONFIG)
1617 port->socket_id = port_numa[pid];
1619 port->socket_id = rte_eth_dev_socket_id(pid);
1622 * if socket_id is invalid,
1623 * set to the first available socket.
1625 if (check_socket_id(port->socket_id) < 0)
1626 port->socket_id = socket_ids[0];
1630 if (socket_num == UMA_NO_CONFIG)
1631 port->socket_id = 0;
1633 port->socket_id = socket_num;
1637 q = RTE_MAX(nb_rxq, nb_txq);
1639 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1642 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1643 if (nb_fwd_streams_new == nb_fwd_streams)
1646 if (fwd_streams != NULL) {
1647 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1648 if (fwd_streams[sm_id] == NULL)
1650 rte_free(fwd_streams[sm_id]);
1651 fwd_streams[sm_id] = NULL;
1653 rte_free(fwd_streams);
1658 nb_fwd_streams = nb_fwd_streams_new;
1659 if (nb_fwd_streams) {
1660 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1661 sizeof(struct fwd_stream *) * nb_fwd_streams,
1662 RTE_CACHE_LINE_SIZE);
1663 if (fwd_streams == NULL)
1664 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1665 " (struct fwd_stream *)) failed\n",
1668 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1669 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1670 " struct fwd_stream", sizeof(struct fwd_stream),
1671 RTE_CACHE_LINE_SIZE);
1672 if (fwd_streams[sm_id] == NULL)
1673 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1674 "(struct fwd_stream) failed\n");
1681 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1683 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1685 uint64_t total_burst, sburst;
1687 uint64_t burst_stats[4];
1688 uint16_t pktnb_stats[4];
1690 int burst_percent[4], sburstp;
1694 * First compute the total number of packet bursts and the
1695 * two highest numbers of bursts of the same number of packets.
1697 memset(&burst_stats, 0x0, sizeof(burst_stats));
1698 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1700 /* Show stats for 0 burst size always */
1701 total_burst = pbs->pkt_burst_spread[0];
1702 burst_stats[0] = pbs->pkt_burst_spread[0];
1705 /* Find the next 2 burst sizes with highest occurrences. */
1706 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1707 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1712 total_burst += nb_burst;
1714 if (nb_burst > burst_stats[1]) {
1715 burst_stats[2] = burst_stats[1];
1716 pktnb_stats[2] = pktnb_stats[1];
1717 burst_stats[1] = nb_burst;
1718 pktnb_stats[1] = nb_pkt;
1719 } else if (nb_burst > burst_stats[2]) {
1720 burst_stats[2] = nb_burst;
1721 pktnb_stats[2] = nb_pkt;
1724 if (total_burst == 0)
1727 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1728 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1730 printf("%d%% of other]\n", 100 - sburstp);
1734 sburst += burst_stats[i];
1735 if (sburst == total_burst) {
1736 printf("%d%% of %d pkts]\n",
1737 100 - sburstp, (int) pktnb_stats[i]);
1742 (double)burst_stats[i] / total_burst * 100;
1743 printf("%d%% of %d pkts + ",
1744 burst_percent[i], (int) pktnb_stats[i]);
1745 sburstp += burst_percent[i];
1748 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1751 fwd_stream_stats_display(streamid_t stream_id)
1753 struct fwd_stream *fs;
1754 static const char *fwd_top_stats_border = "-------";
1756 fs = fwd_streams[stream_id];
1757 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1758 (fs->fwd_dropped == 0))
1760 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1761 "TX Port=%2d/Queue=%2d %s\n",
1762 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1763 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1764 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1765 " TX-dropped: %-14"PRIu64,
1766 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1768 /* if checksum mode */
1769 if (cur_fwd_eng == &csum_fwd_engine) {
1770 printf(" RX- bad IP checksum: %-14"PRIu64
1771 " Rx- bad L4 checksum: %-14"PRIu64
1772 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1773 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1774 fs->rx_bad_outer_l4_csum);
1779 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1780 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1781 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1786 fwd_stats_display(void)
1788 static const char *fwd_stats_border = "----------------------";
1789 static const char *acc_stats_border = "+++++++++++++++";
1791 struct fwd_stream *rx_stream;
1792 struct fwd_stream *tx_stream;
1793 uint64_t tx_dropped;
1794 uint64_t rx_bad_ip_csum;
1795 uint64_t rx_bad_l4_csum;
1796 uint64_t rx_bad_outer_l4_csum;
1797 } ports_stats[RTE_MAX_ETHPORTS];
1798 uint64_t total_rx_dropped = 0;
1799 uint64_t total_tx_dropped = 0;
1800 uint64_t total_rx_nombuf = 0;
1801 struct rte_eth_stats stats;
1802 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1803 uint64_t fwd_cycles = 0;
1805 uint64_t total_recv = 0;
1806 uint64_t total_xmit = 0;
1807 struct rte_port *port;
1812 memset(ports_stats, 0, sizeof(ports_stats));
1814 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1815 struct fwd_stream *fs = fwd_streams[sm_id];
1817 if (cur_fwd_config.nb_fwd_streams >
1818 cur_fwd_config.nb_fwd_ports) {
1819 fwd_stream_stats_display(sm_id);
1821 ports_stats[fs->tx_port].tx_stream = fs;
1822 ports_stats[fs->rx_port].rx_stream = fs;
1825 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1827 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1828 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1829 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1830 fs->rx_bad_outer_l4_csum;
1832 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1833 fwd_cycles += fs->core_cycles;
1836 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1839 pt_id = fwd_ports_ids[i];
1840 port = &ports[pt_id];
1842 rte_eth_stats_get(pt_id, &stats);
1843 stats.ipackets -= port->stats.ipackets;
1844 stats.opackets -= port->stats.opackets;
1845 stats.ibytes -= port->stats.ibytes;
1846 stats.obytes -= port->stats.obytes;
1847 stats.imissed -= port->stats.imissed;
1848 stats.oerrors -= port->stats.oerrors;
1849 stats.rx_nombuf -= port->stats.rx_nombuf;
1851 total_recv += stats.ipackets;
1852 total_xmit += stats.opackets;
1853 total_rx_dropped += stats.imissed;
1854 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1855 total_tx_dropped += stats.oerrors;
1856 total_rx_nombuf += stats.rx_nombuf;
1858 printf("\n %s Forward statistics for port %-2d %s\n",
1859 fwd_stats_border, pt_id, fwd_stats_border);
1861 if (!port->rx_queue_stats_mapping_enabled &&
1862 !port->tx_queue_stats_mapping_enabled) {
1863 printf(" RX-packets: %-14"PRIu64
1864 " RX-dropped: %-14"PRIu64
1865 "RX-total: %-"PRIu64"\n",
1866 stats.ipackets, stats.imissed,
1867 stats.ipackets + stats.imissed);
1869 if (cur_fwd_eng == &csum_fwd_engine)
1870 printf(" Bad-ipcsum: %-14"PRIu64
1871 " Bad-l4csum: %-14"PRIu64
1872 "Bad-outer-l4csum: %-14"PRIu64"\n",
1873 ports_stats[pt_id].rx_bad_ip_csum,
1874 ports_stats[pt_id].rx_bad_l4_csum,
1875 ports_stats[pt_id].rx_bad_outer_l4_csum);
1876 if (stats.ierrors + stats.rx_nombuf > 0) {
1877 printf(" RX-error: %-"PRIu64"\n",
1879 printf(" RX-nombufs: %-14"PRIu64"\n",
1883 printf(" TX-packets: %-14"PRIu64
1884 " TX-dropped: %-14"PRIu64
1885 "TX-total: %-"PRIu64"\n",
1886 stats.opackets, ports_stats[pt_id].tx_dropped,
1887 stats.opackets + ports_stats[pt_id].tx_dropped);
1889 printf(" RX-packets: %14"PRIu64
1890 " RX-dropped:%14"PRIu64
1891 " RX-total:%14"PRIu64"\n",
1892 stats.ipackets, stats.imissed,
1893 stats.ipackets + stats.imissed);
1895 if (cur_fwd_eng == &csum_fwd_engine)
1896 printf(" Bad-ipcsum:%14"PRIu64
1897 " Bad-l4csum:%14"PRIu64
1898 " Bad-outer-l4csum: %-14"PRIu64"\n",
1899 ports_stats[pt_id].rx_bad_ip_csum,
1900 ports_stats[pt_id].rx_bad_l4_csum,
1901 ports_stats[pt_id].rx_bad_outer_l4_csum);
1902 if ((stats.ierrors + stats.rx_nombuf) > 0) {
1903 printf(" RX-error:%"PRIu64"\n", stats.ierrors);
1904 printf(" RX-nombufs: %14"PRIu64"\n",
1908 printf(" TX-packets: %14"PRIu64
1909 " TX-dropped:%14"PRIu64
1910 " TX-total:%14"PRIu64"\n",
1911 stats.opackets, ports_stats[pt_id].tx_dropped,
1912 stats.opackets + ports_stats[pt_id].tx_dropped);
1915 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1916 if (ports_stats[pt_id].rx_stream)
1917 pkt_burst_stats_display("RX",
1918 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1919 if (ports_stats[pt_id].tx_stream)
1920 pkt_burst_stats_display("TX",
1921 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1924 if (port->rx_queue_stats_mapping_enabled) {
1926 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1927 printf(" Stats reg %2d RX-packets:%14"PRIu64
1928 " RX-errors:%14"PRIu64
1929 " RX-bytes:%14"PRIu64"\n",
1930 j, stats.q_ipackets[j],
1931 stats.q_errors[j], stats.q_ibytes[j]);
1935 if (port->tx_queue_stats_mapping_enabled) {
1936 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1937 printf(" Stats reg %2d TX-packets:%14"PRIu64
1940 j, stats.q_opackets[j],
1945 printf(" %s--------------------------------%s\n",
1946 fwd_stats_border, fwd_stats_border);
1949 printf("\n %s Accumulated forward statistics for all ports"
1951 acc_stats_border, acc_stats_border);
1952 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1954 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1956 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1957 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1958 if (total_rx_nombuf > 0)
1959 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1960 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
1962 acc_stats_border, acc_stats_border);
1963 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1964 #define CYC_PER_MHZ 1E6
1965 if (total_recv > 0 || total_xmit > 0) {
1966 uint64_t total_pkts = 0;
1967 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1968 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1969 total_pkts = total_xmit;
1971 total_pkts = total_recv;
1973 printf("\n CPU cycles/packet=%.2F (total cycles="
1974 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1976 (double) fwd_cycles / total_pkts,
1977 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1978 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1984 fwd_stats_reset(void)
1990 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1991 pt_id = fwd_ports_ids[i];
1992 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1994 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1995 struct fwd_stream *fs = fwd_streams[sm_id];
1999 fs->fwd_dropped = 0;
2000 fs->rx_bad_ip_csum = 0;
2001 fs->rx_bad_l4_csum = 0;
2002 fs->rx_bad_outer_l4_csum = 0;
2004 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
2005 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2006 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2008 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
2009 fs->core_cycles = 0;
2015 flush_fwd_rx_queues(void)
2017 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2024 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2025 uint64_t timer_period;
2027 /* convert to number of cycles */
2028 timer_period = rte_get_timer_hz(); /* 1 second timeout */
2030 for (j = 0; j < 2; j++) {
2031 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2032 for (rxq = 0; rxq < nb_rxq; rxq++) {
2033 port_id = fwd_ports_ids[rxp];
2035 * testpmd can stuck in the below do while loop
2036 * if rte_eth_rx_burst() always returns nonzero
2037 * packets. So timer is added to exit this loop
2038 * after 1sec timer expiry.
2040 prev_tsc = rte_rdtsc();
2042 nb_rx = rte_eth_rx_burst(port_id, rxq,
2043 pkts_burst, MAX_PKT_BURST);
2044 for (i = 0; i < nb_rx; i++)
2045 rte_pktmbuf_free(pkts_burst[i]);
2047 cur_tsc = rte_rdtsc();
2048 diff_tsc = cur_tsc - prev_tsc;
2049 timer_tsc += diff_tsc;
2050 } while ((nb_rx > 0) &&
2051 (timer_tsc < timer_period));
2055 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2060 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2062 struct fwd_stream **fsm;
2065 #ifdef RTE_LIBRTE_BITRATE
2066 uint64_t tics_per_1sec;
2067 uint64_t tics_datum;
2068 uint64_t tics_current;
2069 uint16_t i, cnt_ports;
2071 cnt_ports = nb_ports;
2072 tics_datum = rte_rdtsc();
2073 tics_per_1sec = rte_get_timer_hz();
2075 fsm = &fwd_streams[fc->stream_idx];
2076 nb_fs = fc->stream_nb;
2078 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2079 (*pkt_fwd)(fsm[sm_id]);
2080 #ifdef RTE_LIBRTE_BITRATE
2081 if (bitrate_enabled != 0 &&
2082 bitrate_lcore_id == rte_lcore_id()) {
2083 tics_current = rte_rdtsc();
2084 if (tics_current - tics_datum >= tics_per_1sec) {
2085 /* Periodic bitrate calculation */
2086 for (i = 0; i < cnt_ports; i++)
2087 rte_stats_bitrate_calc(bitrate_data,
2089 tics_datum = tics_current;
2093 #ifdef RTE_LIBRTE_LATENCY_STATS
2094 if (latencystats_enabled != 0 &&
2095 latencystats_lcore_id == rte_lcore_id())
2096 rte_latencystats_update();
2099 } while (! fc->stopped);
2103 start_pkt_forward_on_core(void *fwd_arg)
2105 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2106 cur_fwd_config.fwd_eng->packet_fwd);
2111 * Run the TXONLY packet forwarding engine to send a single burst of packets.
2112 * Used to start communication flows in network loopback test configurations.
2115 run_one_txonly_burst_on_core(void *fwd_arg)
2117 struct fwd_lcore *fwd_lc;
2118 struct fwd_lcore tmp_lcore;
2120 fwd_lc = (struct fwd_lcore *) fwd_arg;
2121 tmp_lcore = *fwd_lc;
2122 tmp_lcore.stopped = 1;
2123 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2128 * Launch packet forwarding:
2129 * - Setup per-port forwarding context.
2130 * - launch logical cores with their forwarding configuration.
2133 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2135 port_fwd_begin_t port_fwd_begin;
2140 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2141 if (port_fwd_begin != NULL) {
2142 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2143 (*port_fwd_begin)(fwd_ports_ids[i]);
2145 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2146 lc_id = fwd_lcores_cpuids[i];
2147 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2148 fwd_lcores[i]->stopped = 0;
2149 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2150 fwd_lcores[i], lc_id);
2152 printf("launch lcore %u failed - diag=%d\n",
2159 * Launch packet forwarding configuration.
2162 start_packet_forwarding(int with_tx_first)
2164 port_fwd_begin_t port_fwd_begin;
2165 port_fwd_end_t port_fwd_end;
2166 struct rte_port *port;
2170 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2171 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2173 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2174 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2176 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2177 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2178 (!nb_rxq || !nb_txq))
2179 rte_exit(EXIT_FAILURE,
2180 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2181 cur_fwd_eng->fwd_mode_name);
2183 if (all_ports_started() == 0) {
2184 printf("Not all ports were started\n");
2187 if (test_done == 0) {
2188 printf("Packet forwarding already started\n");
2194 for (i = 0; i < nb_fwd_ports; i++) {
2195 pt_id = fwd_ports_ids[i];
2196 port = &ports[pt_id];
2197 if (!port->dcb_flag) {
2198 printf("In DCB mode, all forwarding ports must "
2199 "be configured in this mode.\n");
2203 if (nb_fwd_lcores == 1) {
2204 printf("In DCB mode,the nb forwarding cores "
2205 "should be larger than 1.\n");
2214 flush_fwd_rx_queues();
2216 pkt_fwd_config_display(&cur_fwd_config);
2217 rxtx_config_display();
2220 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2221 pt_id = fwd_ports_ids[i];
2222 port = &ports[pt_id];
2223 map_port_queue_stats_mapping_registers(pt_id, port);
2225 if (with_tx_first) {
2226 port_fwd_begin = tx_only_engine.port_fwd_begin;
2227 if (port_fwd_begin != NULL) {
2228 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2229 (*port_fwd_begin)(fwd_ports_ids[i]);
2231 while (with_tx_first--) {
2232 launch_packet_forwarding(
2233 run_one_txonly_burst_on_core);
2234 rte_eal_mp_wait_lcore();
2236 port_fwd_end = tx_only_engine.port_fwd_end;
2237 if (port_fwd_end != NULL) {
2238 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2239 (*port_fwd_end)(fwd_ports_ids[i]);
2242 launch_packet_forwarding(start_pkt_forward_on_core);
2246 stop_packet_forwarding(void)
2248 port_fwd_end_t port_fwd_end;
2254 printf("Packet forwarding not started\n");
2257 printf("Telling cores to stop...");
2258 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2259 fwd_lcores[lc_id]->stopped = 1;
2260 printf("\nWaiting for lcores to finish...\n");
2261 rte_eal_mp_wait_lcore();
2262 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2263 if (port_fwd_end != NULL) {
2264 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2265 pt_id = fwd_ports_ids[i];
2266 (*port_fwd_end)(pt_id);
2270 fwd_stats_display();
2272 printf("\nDone.\n");
2277 dev_set_link_up(portid_t pid)
2279 if (rte_eth_dev_set_link_up(pid) < 0)
2280 printf("\nSet link up fail.\n");
2284 dev_set_link_down(portid_t pid)
2286 if (rte_eth_dev_set_link_down(pid) < 0)
2287 printf("\nSet link down fail.\n");
2291 all_ports_started(void)
2294 struct rte_port *port;
2296 RTE_ETH_FOREACH_DEV(pi) {
2298 /* Check if there is a port which is not started */
2299 if ((port->port_status != RTE_PORT_STARTED) &&
2300 (port->slave_flag == 0))
2304 /* No port is not started */
2309 port_is_stopped(portid_t port_id)
2311 struct rte_port *port = &ports[port_id];
2313 if ((port->port_status != RTE_PORT_STOPPED) &&
2314 (port->slave_flag == 0))
2320 all_ports_stopped(void)
2324 RTE_ETH_FOREACH_DEV(pi) {
2325 if (!port_is_stopped(pi))
2333 port_is_started(portid_t port_id)
2335 if (port_id_is_invalid(port_id, ENABLED_WARN))
2338 if (ports[port_id].port_status != RTE_PORT_STARTED)
2344 /* Configure the Rx and Tx hairpin queues for the selected port. */
2346 setup_hairpin_queues(portid_t pi)
2349 struct rte_eth_hairpin_conf hairpin_conf = {
2354 struct rte_port *port = &ports[pi];
2356 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2357 hairpin_conf.peers[0].port = pi;
2358 hairpin_conf.peers[0].queue = i + nb_rxq;
2359 diag = rte_eth_tx_hairpin_queue_setup
2360 (pi, qi, nb_txd, &hairpin_conf);
2365 /* Fail to setup rx queue, return */
2366 if (rte_atomic16_cmpset(&(port->port_status),
2368 RTE_PORT_STOPPED) == 0)
2369 printf("Port %d can not be set back "
2370 "to stopped\n", pi);
2371 printf("Fail to configure port %d hairpin "
2373 /* try to reconfigure queues next time */
2374 port->need_reconfig_queues = 1;
2377 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2378 hairpin_conf.peers[0].port = pi;
2379 hairpin_conf.peers[0].queue = i + nb_txq;
2380 diag = rte_eth_rx_hairpin_queue_setup
2381 (pi, qi, nb_rxd, &hairpin_conf);
2386 /* Fail to setup rx queue, return */
2387 if (rte_atomic16_cmpset(&(port->port_status),
2389 RTE_PORT_STOPPED) == 0)
2390 printf("Port %d can not be set back "
2391 "to stopped\n", pi);
2392 printf("Fail to configure port %d hairpin "
2394 /* try to reconfigure queues next time */
2395 port->need_reconfig_queues = 1;
2402 start_port(portid_t pid)
2404 int diag, need_check_link_status = -1;
2407 struct rte_port *port;
2408 struct rte_ether_addr mac_addr;
2409 struct rte_eth_hairpin_cap cap;
2411 if (port_id_is_invalid(pid, ENABLED_WARN))
2416 RTE_ETH_FOREACH_DEV(pi) {
2417 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2420 need_check_link_status = 0;
2422 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2423 RTE_PORT_HANDLING) == 0) {
2424 printf("Port %d is now not stopped\n", pi);
2428 if (port->need_reconfig > 0) {
2429 port->need_reconfig = 0;
2431 if (flow_isolate_all) {
2432 int ret = port_flow_isolate(pi, 1);
2434 printf("Failed to apply isolated"
2435 " mode on port %d\n", pi);
2439 configure_rxtx_dump_callbacks(0);
2440 printf("Configuring Port %d (socket %u)\n", pi,
2442 if (nb_hairpinq > 0 &&
2443 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2444 printf("Port %d doesn't support hairpin "
2448 /* configure port */
2449 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2450 nb_txq + nb_hairpinq,
2453 if (rte_atomic16_cmpset(&(port->port_status),
2454 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2455 printf("Port %d can not be set back "
2456 "to stopped\n", pi);
2457 printf("Fail to configure port %d\n", pi);
2458 /* try to reconfigure port next time */
2459 port->need_reconfig = 1;
2463 if (port->need_reconfig_queues > 0) {
2464 port->need_reconfig_queues = 0;
2465 /* setup tx queues */
2466 for (qi = 0; qi < nb_txq; qi++) {
2467 if ((numa_support) &&
2468 (txring_numa[pi] != NUMA_NO_CONFIG))
2469 diag = rte_eth_tx_queue_setup(pi, qi,
2470 port->nb_tx_desc[qi],
2472 &(port->tx_conf[qi]));
2474 diag = rte_eth_tx_queue_setup(pi, qi,
2475 port->nb_tx_desc[qi],
2477 &(port->tx_conf[qi]));
2482 /* Fail to setup tx queue, return */
2483 if (rte_atomic16_cmpset(&(port->port_status),
2485 RTE_PORT_STOPPED) == 0)
2486 printf("Port %d can not be set back "
2487 "to stopped\n", pi);
2488 printf("Fail to configure port %d tx queues\n",
2490 /* try to reconfigure queues next time */
2491 port->need_reconfig_queues = 1;
2494 for (qi = 0; qi < nb_rxq; qi++) {
2495 /* setup rx queues */
2496 if ((numa_support) &&
2497 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2498 struct rte_mempool * mp =
2499 mbuf_pool_find(rxring_numa[pi]);
2501 printf("Failed to setup RX queue:"
2502 "No mempool allocation"
2503 " on the socket %d\n",
2508 diag = rte_eth_rx_queue_setup(pi, qi,
2509 port->nb_rx_desc[qi],
2511 &(port->rx_conf[qi]),
2514 struct rte_mempool *mp =
2515 mbuf_pool_find(port->socket_id);
2517 printf("Failed to setup RX queue:"
2518 "No mempool allocation"
2519 " on the socket %d\n",
2523 diag = rte_eth_rx_queue_setup(pi, qi,
2524 port->nb_rx_desc[qi],
2526 &(port->rx_conf[qi]),
2532 /* Fail to setup rx queue, return */
2533 if (rte_atomic16_cmpset(&(port->port_status),
2535 RTE_PORT_STOPPED) == 0)
2536 printf("Port %d can not be set back "
2537 "to stopped\n", pi);
2538 printf("Fail to configure port %d rx queues\n",
2540 /* try to reconfigure queues next time */
2541 port->need_reconfig_queues = 1;
2544 /* setup hairpin queues */
2545 if (setup_hairpin_queues(pi) != 0)
2548 configure_rxtx_dump_callbacks(verbose_level);
2550 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2554 "Port %d: Failed to disable Ptype parsing\n",
2559 if (rte_eth_dev_start(pi) < 0) {
2560 printf("Fail to start port %d\n", pi);
2562 /* Fail to setup rx queue, return */
2563 if (rte_atomic16_cmpset(&(port->port_status),
2564 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2565 printf("Port %d can not be set back to "
2570 if (rte_atomic16_cmpset(&(port->port_status),
2571 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2572 printf("Port %d can not be set into started\n", pi);
2574 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2575 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2576 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2577 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2578 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2580 /* at least one port started, need checking link status */
2581 need_check_link_status = 1;
2584 if (need_check_link_status == 1 && !no_link_check)
2585 check_all_ports_link_status(RTE_PORT_ALL);
2586 else if (need_check_link_status == 0)
2587 printf("Please stop the ports first\n");
2594 stop_port(portid_t pid)
2597 struct rte_port *port;
2598 int need_check_link_status = 0;
2605 if (port_id_is_invalid(pid, ENABLED_WARN))
2608 printf("Stopping ports...\n");
2610 RTE_ETH_FOREACH_DEV(pi) {
2611 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2614 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2615 printf("Please remove port %d from forwarding configuration.\n", pi);
2619 if (port_is_bonding_slave(pi)) {
2620 printf("Please remove port %d from bonded device.\n", pi);
2625 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2626 RTE_PORT_HANDLING) == 0)
2629 rte_eth_dev_stop(pi);
2631 if (rte_atomic16_cmpset(&(port->port_status),
2632 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2633 printf("Port %d can not be set into stopped\n", pi);
2634 need_check_link_status = 1;
2636 if (need_check_link_status && !no_link_check)
2637 check_all_ports_link_status(RTE_PORT_ALL);
2643 remove_invalid_ports_in(portid_t *array, portid_t *total)
2646 portid_t new_total = 0;
2648 for (i = 0; i < *total; i++)
2649 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2650 array[new_total] = array[i];
2657 remove_invalid_ports(void)
2659 remove_invalid_ports_in(ports_ids, &nb_ports);
2660 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2661 nb_cfg_ports = nb_fwd_ports;
2665 close_port(portid_t pid)
2668 struct rte_port *port;
2670 if (port_id_is_invalid(pid, ENABLED_WARN))
2673 printf("Closing ports...\n");
2675 RTE_ETH_FOREACH_DEV(pi) {
2676 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2679 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2680 printf("Please remove port %d from forwarding configuration.\n", pi);
2684 if (port_is_bonding_slave(pi)) {
2685 printf("Please remove port %d from bonded device.\n", pi);
2690 if (rte_atomic16_cmpset(&(port->port_status),
2691 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2692 printf("Port %d is already closed\n", pi);
2696 if (rte_atomic16_cmpset(&(port->port_status),
2697 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2698 printf("Port %d is now not stopped\n", pi);
2702 if (port->flow_list)
2703 port_flow_flush(pi);
2704 rte_eth_dev_close(pi);
2706 remove_invalid_ports();
2708 if (rte_atomic16_cmpset(&(port->port_status),
2709 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2710 printf("Port %d cannot be set to closed\n", pi);
2717 reset_port(portid_t pid)
2721 struct rte_port *port;
2723 if (port_id_is_invalid(pid, ENABLED_WARN))
2726 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2727 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2728 printf("Can not reset port(s), please stop port(s) first.\n");
2732 printf("Resetting ports...\n");
2734 RTE_ETH_FOREACH_DEV(pi) {
2735 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2738 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2739 printf("Please remove port %d from forwarding "
2740 "configuration.\n", pi);
2744 if (port_is_bonding_slave(pi)) {
2745 printf("Please remove port %d from bonded device.\n",
2750 diag = rte_eth_dev_reset(pi);
2753 port->need_reconfig = 1;
2754 port->need_reconfig_queues = 1;
2756 printf("Failed to reset port %d. diag=%d\n", pi, diag);
2764 attach_port(char *identifier)
2767 struct rte_dev_iterator iterator;
2769 printf("Attaching a new port...\n");
2771 if (identifier == NULL) {
2772 printf("Invalid parameters are specified\n");
2776 if (rte_dev_probe(identifier) < 0) {
2777 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2781 /* first attach mode: event */
2782 if (setup_on_probe_event) {
2783 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2784 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2785 if (ports[pi].port_status == RTE_PORT_HANDLING &&
2786 ports[pi].need_setup != 0)
2787 setup_attached_port(pi);
2791 /* second attach mode: iterator */
2792 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2793 /* setup ports matching the devargs used for probing */
2794 if (port_is_forwarding(pi))
2795 continue; /* port was already attached before */
2796 setup_attached_port(pi);
2801 setup_attached_port(portid_t pi)
2803 unsigned int socket_id;
2806 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2807 /* if socket_id is invalid, set to the first available socket. */
2808 if (check_socket_id(socket_id) < 0)
2809 socket_id = socket_ids[0];
2810 reconfig(pi, socket_id);
2811 ret = rte_eth_promiscuous_enable(pi);
2813 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2814 pi, rte_strerror(-ret));
2816 ports_ids[nb_ports++] = pi;
2817 fwd_ports_ids[nb_fwd_ports++] = pi;
2818 nb_cfg_ports = nb_fwd_ports;
2819 ports[pi].need_setup = 0;
2820 ports[pi].port_status = RTE_PORT_STOPPED;
2822 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2827 detach_device(struct rte_device *dev)
2832 printf("Device already removed\n");
2836 printf("Removing a device...\n");
2838 if (rte_dev_remove(dev) < 0) {
2839 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2842 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2843 /* reset mapping between old ports and removed device */
2844 rte_eth_devices[sibling].device = NULL;
2845 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2846 /* sibling ports are forced to be closed */
2847 ports[sibling].port_status = RTE_PORT_CLOSED;
2848 printf("Port %u is closed\n", sibling);
2852 remove_invalid_ports();
2854 printf("Device is detached\n");
2855 printf("Now total ports is %d\n", nb_ports);
2861 detach_port_device(portid_t port_id)
2863 if (port_id_is_invalid(port_id, ENABLED_WARN))
2866 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2867 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2868 printf("Port not stopped\n");
2871 printf("Port was not closed\n");
2872 if (ports[port_id].flow_list)
2873 port_flow_flush(port_id);
2876 detach_device(rte_eth_devices[port_id].device);
2880 detach_devargs(char *identifier)
2882 struct rte_dev_iterator iterator;
2883 struct rte_devargs da;
2886 printf("Removing a device...\n");
2888 memset(&da, 0, sizeof(da));
2889 if (rte_devargs_parsef(&da, "%s", identifier)) {
2890 printf("cannot parse identifier\n");
2896 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2897 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2898 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2899 printf("Port %u not stopped\n", port_id);
2900 rte_eth_iterator_cleanup(&iterator);
2904 /* sibling ports are forced to be closed */
2905 if (ports[port_id].flow_list)
2906 port_flow_flush(port_id);
2907 ports[port_id].port_status = RTE_PORT_CLOSED;
2908 printf("Port %u is now closed\n", port_id);
2912 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2913 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2914 da.name, da.bus->name);
2918 remove_invalid_ports();
2920 printf("Device %s is detached\n", identifier);
2921 printf("Now total ports is %d\n", nb_ports);
2933 stop_packet_forwarding();
2935 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2937 if (mp_alloc_type == MP_ALLOC_ANON)
2938 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2942 if (ports != NULL) {
2944 RTE_ETH_FOREACH_DEV(pt_id) {
2945 printf("\nStopping port %d...\n", pt_id);
2949 RTE_ETH_FOREACH_DEV(pt_id) {
2950 printf("\nShutting down port %d...\n", pt_id);
2957 ret = rte_dev_event_monitor_stop();
2960 "fail to stop device event monitor.");
2964 ret = rte_dev_event_callback_unregister(NULL,
2965 dev_event_callback, NULL);
2968 "fail to unregister device event callback.\n");
2972 ret = rte_dev_hotplug_handle_disable();
2975 "fail to disable hotplug handling.\n");
2979 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2981 rte_mempool_free(mempools[i]);
2984 printf("\nBye...\n");
2987 typedef void (*cmd_func_t)(void);
2988 struct pmd_test_command {
2989 const char *cmd_name;
2990 cmd_func_t cmd_func;
2993 /* Check the link status of all ports in up to 9s, and print them finally */
2995 check_all_ports_link_status(uint32_t port_mask)
2997 #define CHECK_INTERVAL 100 /* 100ms */
2998 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3000 uint8_t count, all_ports_up, print_flag = 0;
3001 struct rte_eth_link link;
3004 printf("Checking link statuses...\n");
3006 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3008 RTE_ETH_FOREACH_DEV(portid) {
3009 if ((port_mask & (1 << portid)) == 0)
3011 memset(&link, 0, sizeof(link));
3012 ret = rte_eth_link_get_nowait(portid, &link);
3015 if (print_flag == 1)
3016 printf("Port %u link get failed: %s\n",
3017 portid, rte_strerror(-ret));
3020 /* print link status if flag set */
3021 if (print_flag == 1) {
3022 if (link.link_status)
3024 "Port%d Link Up. speed %u Mbps- %s\n",
3025 portid, link.link_speed,
3026 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3027 ("full-duplex") : ("half-duplex"));
3029 printf("Port %d Link Down\n", portid);
3032 /* clear all_ports_up flag if any link down */
3033 if (link.link_status == ETH_LINK_DOWN) {
3038 /* after finally printing all link status, get out */
3039 if (print_flag == 1)
3042 if (all_ports_up == 0) {
3044 rte_delay_ms(CHECK_INTERVAL);
3047 /* set the print_flag if all ports up or timeout */
3048 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3058 * This callback is for remove a port for a device. It has limitation because
3059 * it is not for multiple port removal for a device.
3060 * TODO: the device detach invoke will plan to be removed from user side to
3061 * eal. And convert all PMDs to free port resources on ether device closing.
3064 rmv_port_callback(void *arg)
3066 int need_to_start = 0;
3067 int org_no_link_check = no_link_check;
3068 portid_t port_id = (intptr_t)arg;
3069 struct rte_device *dev;
3071 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3073 if (!test_done && port_is_forwarding(port_id)) {
3075 stop_packet_forwarding();
3079 no_link_check = org_no_link_check;
3081 /* Save rte_device pointer before closing ethdev port */
3082 dev = rte_eth_devices[port_id].device;
3083 close_port(port_id);
3084 detach_device(dev); /* might be already removed or have more ports */
3087 start_packet_forwarding(0);
3090 /* This function is used by the interrupt thread */
3092 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3095 RTE_SET_USED(param);
3096 RTE_SET_USED(ret_param);
3098 if (type >= RTE_ETH_EVENT_MAX) {
3099 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3100 port_id, __func__, type);
3102 } else if (event_print_mask & (UINT32_C(1) << type)) {
3103 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3104 eth_event_desc[type]);
3109 case RTE_ETH_EVENT_NEW:
3110 ports[port_id].need_setup = 1;
3111 ports[port_id].port_status = RTE_PORT_HANDLING;
3113 case RTE_ETH_EVENT_INTR_RMV:
3114 if (port_id_is_invalid(port_id, DISABLED_WARN))
3116 if (rte_eal_alarm_set(100000,
3117 rmv_port_callback, (void *)(intptr_t)port_id))
3118 fprintf(stderr, "Could not set up deferred device removal\n");
3127 register_eth_event_callback(void)
3130 enum rte_eth_event_type event;
3132 for (event = RTE_ETH_EVENT_UNKNOWN;
3133 event < RTE_ETH_EVENT_MAX; event++) {
3134 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3139 TESTPMD_LOG(ERR, "Failed to register callback for "
3140 "%s event\n", eth_event_desc[event]);
3148 /* This function is used by the interrupt thread */
3150 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3151 __rte_unused void *arg)
3156 if (type >= RTE_DEV_EVENT_MAX) {
3157 fprintf(stderr, "%s called upon invalid event %d\n",
3163 case RTE_DEV_EVENT_REMOVE:
3164 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3166 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3168 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3173 * Because the user's callback is invoked in eal interrupt
3174 * callback, the interrupt callback need to be finished before
3175 * it can be unregistered when detaching device. So finish
3176 * callback soon and use a deferred removal to detach device
3177 * is need. It is a workaround, once the device detaching be
3178 * moved into the eal in the future, the deferred removal could
3181 if (rte_eal_alarm_set(100000,
3182 rmv_port_callback, (void *)(intptr_t)port_id))
3184 "Could not set up deferred device removal\n");
3186 case RTE_DEV_EVENT_ADD:
3187 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3189 /* TODO: After finish kernel driver binding,
3190 * begin to attach port.
3199 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3203 uint8_t mapping_found = 0;
3205 for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3206 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3207 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3208 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3209 tx_queue_stats_mappings[i].queue_id,
3210 tx_queue_stats_mappings[i].stats_counter_id);
3217 port->tx_queue_stats_mapping_enabled = 1;
3222 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3226 uint8_t mapping_found = 0;
3228 for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3229 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3230 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3231 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3232 rx_queue_stats_mappings[i].queue_id,
3233 rx_queue_stats_mappings[i].stats_counter_id);
3240 port->rx_queue_stats_mapping_enabled = 1;
3245 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3249 diag = set_tx_queue_stats_mapping_registers(pi, port);
3251 if (diag == -ENOTSUP) {
3252 port->tx_queue_stats_mapping_enabled = 0;
3253 printf("TX queue stats mapping not supported port id=%d\n", pi);
3256 rte_exit(EXIT_FAILURE,
3257 "set_tx_queue_stats_mapping_registers "
3258 "failed for port id=%d diag=%d\n",
3262 diag = set_rx_queue_stats_mapping_registers(pi, port);
3264 if (diag == -ENOTSUP) {
3265 port->rx_queue_stats_mapping_enabled = 0;
3266 printf("RX queue stats mapping not supported port id=%d\n", pi);
3269 rte_exit(EXIT_FAILURE,
3270 "set_rx_queue_stats_mapping_registers "
3271 "failed for port id=%d diag=%d\n",
3277 rxtx_port_config(struct rte_port *port)
3282 for (qid = 0; qid < nb_rxq; qid++) {
3283 offloads = port->rx_conf[qid].offloads;
3284 port->rx_conf[qid] = port->dev_info.default_rxconf;
3286 port->rx_conf[qid].offloads = offloads;
3288 /* Check if any Rx parameters have been passed */
3289 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3290 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3292 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3293 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3295 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3296 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3298 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3299 port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3301 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3302 port->rx_conf[qid].rx_drop_en = rx_drop_en;
3304 port->nb_rx_desc[qid] = nb_rxd;
3307 for (qid = 0; qid < nb_txq; qid++) {
3308 offloads = port->tx_conf[qid].offloads;
3309 port->tx_conf[qid] = port->dev_info.default_txconf;
3311 port->tx_conf[qid].offloads = offloads;
3313 /* Check if any Tx parameters have been passed */
3314 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3315 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3317 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3318 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3320 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3321 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3323 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3324 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3326 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3327 port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3329 port->nb_tx_desc[qid] = nb_txd;
3334 init_port_config(void)
3337 struct rte_port *port;
3340 RTE_ETH_FOREACH_DEV(pid) {
3342 port->dev_conf.fdir_conf = fdir_conf;
3344 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3349 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3350 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3351 rss_hf & port->dev_info.flow_type_rss_offloads;
3353 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3354 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3357 if (port->dcb_flag == 0) {
3358 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3359 port->dev_conf.rxmode.mq_mode =
3360 (enum rte_eth_rx_mq_mode)
3361 (rx_mq_mode & ETH_MQ_RX_RSS);
3363 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3366 rxtx_port_config(port);
3368 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3372 map_port_queue_stats_mapping_registers(pid, port);
3373 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3374 rte_pmd_ixgbe_bypass_init(pid);
3377 if (lsc_interrupt &&
3378 (rte_eth_devices[pid].data->dev_flags &
3379 RTE_ETH_DEV_INTR_LSC))
3380 port->dev_conf.intr_conf.lsc = 1;
3381 if (rmv_interrupt &&
3382 (rte_eth_devices[pid].data->dev_flags &
3383 RTE_ETH_DEV_INTR_RMV))
3384 port->dev_conf.intr_conf.rmv = 1;
3388 void set_port_slave_flag(portid_t slave_pid)
3390 struct rte_port *port;
3392 port = &ports[slave_pid];
3393 port->slave_flag = 1;
3396 void clear_port_slave_flag(portid_t slave_pid)
3398 struct rte_port *port;
3400 port = &ports[slave_pid];
3401 port->slave_flag = 0;
3404 uint8_t port_is_bonding_slave(portid_t slave_pid)
3406 struct rte_port *port;
3408 port = &ports[slave_pid];
3409 if ((rte_eth_devices[slave_pid].data->dev_flags &
3410 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3415 const uint16_t vlan_tags[] = {
3416 0, 1, 2, 3, 4, 5, 6, 7,
3417 8, 9, 10, 11, 12, 13, 14, 15,
3418 16, 17, 18, 19, 20, 21, 22, 23,
3419 24, 25, 26, 27, 28, 29, 30, 31
3423 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3424 enum dcb_mode_enable dcb_mode,
3425 enum rte_eth_nb_tcs num_tcs,
3430 struct rte_eth_rss_conf rss_conf;
3433 * Builds up the correct configuration for dcb+vt based on the vlan tags array
3434 * given above, and the number of traffic classes available for use.
3436 if (dcb_mode == DCB_VT_ENABLED) {
3437 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3438 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3439 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3440 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3442 /* VMDQ+DCB RX and TX configurations */
3443 vmdq_rx_conf->enable_default_pool = 0;
3444 vmdq_rx_conf->default_pool = 0;
3445 vmdq_rx_conf->nb_queue_pools =
3446 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3447 vmdq_tx_conf->nb_queue_pools =
3448 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3450 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3451 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3452 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3453 vmdq_rx_conf->pool_map[i].pools =
3454 1 << (i % vmdq_rx_conf->nb_queue_pools);
3456 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3457 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3458 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3461 /* set DCB mode of RX and TX of multiple queues */
3462 eth_conf->rxmode.mq_mode =
3463 (enum rte_eth_rx_mq_mode)
3464 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3465 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3467 struct rte_eth_dcb_rx_conf *rx_conf =
3468 ð_conf->rx_adv_conf.dcb_rx_conf;
3469 struct rte_eth_dcb_tx_conf *tx_conf =
3470 ð_conf->tx_adv_conf.dcb_tx_conf;
3472 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3474 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3478 rx_conf->nb_tcs = num_tcs;
3479 tx_conf->nb_tcs = num_tcs;
3481 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3482 rx_conf->dcb_tc[i] = i % num_tcs;
3483 tx_conf->dcb_tc[i] = i % num_tcs;
3486 eth_conf->rxmode.mq_mode =
3487 (enum rte_eth_rx_mq_mode)
3488 (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3489 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3490 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3494 eth_conf->dcb_capability_en =
3495 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3497 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3503 init_port_dcb_config(portid_t pid,
3504 enum dcb_mode_enable dcb_mode,
3505 enum rte_eth_nb_tcs num_tcs,
3508 struct rte_eth_conf port_conf;
3509 struct rte_port *rte_port;
3513 rte_port = &ports[pid];
3515 memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3516 /* Enter DCB configuration status */
3519 port_conf.rxmode = rte_port->dev_conf.rxmode;
3520 port_conf.txmode = rte_port->dev_conf.txmode;
3522 /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3523 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3526 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3528 /* re-configure the device . */
3529 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3533 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3537 /* If dev_info.vmdq_pool_base is greater than 0,
3538 * the queue id of vmdq pools is started after pf queues.
3540 if (dcb_mode == DCB_VT_ENABLED &&
3541 rte_port->dev_info.vmdq_pool_base > 0) {
3542 printf("VMDQ_DCB multi-queue mode is nonsensical"
3543 " for port %d.", pid);
3547 /* Assume the ports in testpmd have the same dcb capability
3548 * and has the same number of rxq and txq in dcb mode
3550 if (dcb_mode == DCB_VT_ENABLED) {
3551 if (rte_port->dev_info.max_vfs > 0) {
3552 nb_rxq = rte_port->dev_info.nb_rx_queues;
3553 nb_txq = rte_port->dev_info.nb_tx_queues;
3555 nb_rxq = rte_port->dev_info.max_rx_queues;
3556 nb_txq = rte_port->dev_info.max_tx_queues;
3559 /*if vt is disabled, use all pf queues */
3560 if (rte_port->dev_info.vmdq_pool_base == 0) {
3561 nb_rxq = rte_port->dev_info.max_rx_queues;
3562 nb_txq = rte_port->dev_info.max_tx_queues;
3564 nb_rxq = (queueid_t)num_tcs;
3565 nb_txq = (queueid_t)num_tcs;
3569 rx_free_thresh = 64;
3571 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3573 rxtx_port_config(rte_port);
3575 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3576 for (i = 0; i < RTE_DIM(vlan_tags); i++)
3577 rx_vft_set(pid, vlan_tags[i], 1);
3579 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3583 map_port_queue_stats_mapping_registers(pid, rte_port);
3585 rte_port->dcb_flag = 1;
3593 /* Configuration of Ethernet ports. */
3594 ports = rte_zmalloc("testpmd: ports",
3595 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3596 RTE_CACHE_LINE_SIZE);
3597 if (ports == NULL) {
3598 rte_exit(EXIT_FAILURE,
3599 "rte_zmalloc(%d struct rte_port) failed\n",
3603 /* Initialize ports NUMA structures */
3604 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3605 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3606 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3620 const char clr[] = { 27, '[', '2', 'J', '\0' };
3621 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3623 /* Clear screen and move to top left */
3624 printf("%s%s", clr, top_left);
3626 printf("\nPort statistics ====================================");
3627 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3628 nic_stats_display(fwd_ports_ids[i]);
3634 signal_handler(int signum)
3636 if (signum == SIGINT || signum == SIGTERM) {
3637 printf("\nSignal %d received, preparing to exit...\n",
3639 #ifdef RTE_LIBRTE_PDUMP
3640 /* uninitialize packet capture framework */
3643 #ifdef RTE_LIBRTE_LATENCY_STATS
3644 if (latencystats_enabled != 0)
3645 rte_latencystats_uninit();
3648 /* Set flag to indicate the force termination. */
3650 /* exit with the expected status */
3651 signal(signum, SIG_DFL);
3652 kill(getpid(), signum);
3657 main(int argc, char** argv)
3664 signal(SIGINT, signal_handler);
3665 signal(SIGTERM, signal_handler);
3667 testpmd_logtype = rte_log_register("testpmd");
3668 if (testpmd_logtype < 0)
3669 rte_exit(EXIT_FAILURE, "Cannot register log type");
3670 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3672 diag = rte_eal_init(argc, argv);
3674 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3675 rte_strerror(rte_errno));
3677 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3678 rte_exit(EXIT_FAILURE,
3679 "Secondary process type not supported.\n");
3681 ret = register_eth_event_callback();
3683 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3685 #ifdef RTE_LIBRTE_PDUMP
3686 /* initialize packet capture framework */
3691 RTE_ETH_FOREACH_DEV(port_id) {
3692 ports_ids[count] = port_id;
3695 nb_ports = (portid_t) count;
3697 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3699 /* allocate port structures, and init them */
3702 set_def_fwd_config();
3704 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3705 "Check the core mask argument\n");
3707 /* Bitrate/latency stats disabled by default */
3708 #ifdef RTE_LIBRTE_BITRATE
3709 bitrate_enabled = 0;
3711 #ifdef RTE_LIBRTE_LATENCY_STATS
3712 latencystats_enabled = 0;
3715 /* on FreeBSD, mlockall() is disabled by default */
3716 #ifdef RTE_EXEC_ENV_FREEBSD
3725 launch_args_parse(argc, argv);
3727 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3728 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3732 if (tx_first && interactive)
3733 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3734 "interactive mode.\n");
3736 if (tx_first && lsc_interrupt) {
3737 printf("Warning: lsc_interrupt needs to be off when "
3738 " using tx_first. Disabling.\n");
3742 if (!nb_rxq && !nb_txq)
3743 printf("Warning: Either rx or tx queues should be non-zero\n");
3745 if (nb_rxq > 1 && nb_rxq > nb_txq)
3746 printf("Warning: nb_rxq=%d enables RSS configuration, "
3747 "but nb_txq=%d will prevent to fully test it.\n",
3753 ret = rte_dev_hotplug_handle_enable();
3756 "fail to enable hotplug handling.");
3760 ret = rte_dev_event_monitor_start();
3763 "fail to start device event monitoring.");
3767 ret = rte_dev_event_callback_register(NULL,
3768 dev_event_callback, NULL);
3771 "fail to register device event callback\n");
3776 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3777 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3779 /* set all ports to promiscuous mode by default */
3780 RTE_ETH_FOREACH_DEV(port_id) {
3781 ret = rte_eth_promiscuous_enable(port_id);
3783 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3784 port_id, rte_strerror(-ret));
3787 /* Init metrics library */
3788 rte_metrics_init(rte_socket_id());
3790 #ifdef RTE_LIBRTE_LATENCY_STATS
3791 if (latencystats_enabled != 0) {
3792 int ret = rte_latencystats_init(1, NULL);
3794 printf("Warning: latencystats init()"
3795 " returned error %d\n", ret);
3796 printf("Latencystats running on lcore %d\n",
3797 latencystats_lcore_id);
3801 /* Setup bitrate stats */
3802 #ifdef RTE_LIBRTE_BITRATE
3803 if (bitrate_enabled != 0) {
3804 bitrate_data = rte_stats_bitrate_create();
3805 if (bitrate_data == NULL)
3806 rte_exit(EXIT_FAILURE,
3807 "Could not allocate bitrate data.\n");
3808 rte_stats_bitrate_reg(bitrate_data);
3812 #ifdef RTE_LIBRTE_CMDLINE
3813 if (strlen(cmdline_filename) != 0)
3814 cmdline_read_from_file(cmdline_filename);
3816 if (interactive == 1) {
3818 printf("Start automatic packet forwarding\n");
3819 start_packet_forwarding(0);
3831 printf("No commandline core given, start packet forwarding\n");
3832 start_packet_forwarding(tx_first);
3833 if (stats_period != 0) {
3834 uint64_t prev_time = 0, cur_time, diff_time = 0;
3835 uint64_t timer_period;
3837 /* Convert to number of cycles */
3838 timer_period = stats_period * rte_get_timer_hz();
3840 while (f_quit == 0) {
3841 cur_time = rte_get_timer_cycles();
3842 diff_time += cur_time - prev_time;
3844 if (diff_time >= timer_period) {
3846 /* Reset the timer */
3849 /* Sleep to avoid unnecessary checks */
3850 prev_time = cur_time;
3855 printf("Press enter to exit\n");
3856 rc = read(0, &c, 1);
3862 ret = rte_eal_cleanup();
3864 rte_exit(EXIT_FAILURE,
3865 "EAL cleanup failed: %s\n", strerror(-ret));
3867 return EXIT_SUCCESS;