1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
13 #include <sys/types.h>
17 #include <sys/queue.h>
24 #include <rte_common.h>
25 #include <rte_errno.h>
26 #include <rte_byteorder.h>
28 #include <rte_debug.h>
29 #include <rte_cycles.h>
30 #include <rte_memory.h>
31 #include <rte_memcpy.h>
32 #include <rte_launch.h>
34 #include <rte_alarm.h>
35 #include <rte_per_lcore.h>
36 #include <rte_lcore.h>
37 #include <rte_atomic.h>
38 #include <rte_branch_prediction.h>
39 #include <rte_mempool.h>
40 #include <rte_malloc.h>
42 #include <rte_mbuf_pool_ops.h>
43 #include <rte_interrupts.h>
45 #include <rte_ether.h>
46 #include <rte_ethdev.h>
48 #include <rte_string_fns.h>
50 #include <rte_pmd_ixgbe.h>
53 #include <rte_pdump.h>
56 #include <rte_metrics.h>
57 #ifdef RTE_LIB_BITRATESTATS
58 #include <rte_bitrate.h>
60 #ifdef RTE_LIB_LATENCYSTATS
61 #include <rte_latencystats.h>
67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
68 #define HUGE_FLAG (0x40000)
70 #define HUGE_FLAG MAP_HUGETLB
73 #ifndef MAP_HUGE_SHIFT
74 /* older kernels (or FreeBSD) will not have this define */
75 #define HUGE_SHIFT (26)
77 #define HUGE_SHIFT MAP_HUGE_SHIFT
80 #define EXTMEM_HEAP_NAME "extmem"
81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
83 uint16_t verbose_level = 0; /**< Silent by default. */
84 int testpmd_logtype; /**< Log type for testpmd logs */
86 /* use main core for command line ? */
87 uint8_t interactive = 0;
88 uint8_t auto_start = 0;
90 char cmdline_filename[PATH_MAX] = {0};
93 * NUMA support configuration.
94 * When set, the NUMA support attempts to dispatch the allocation of the
95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
96 * probed ports among the CPU sockets 0 and 1.
97 * Otherwise, all memory is allocated from CPU socket 0.
99 uint8_t numa_support = 1; /**< numa enabled by default */
102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is
105 uint8_t socket_num = UMA_NO_CONFIG;
108 * Select mempool allocation type:
109 * - native: use regular DPDK memory
110 * - anon: use regular DPDK memory to create mempool, but populate using
111 * anonymous memory (may not be IOVA-contiguous)
112 * - xmem: use externally allocated hugepage memory
114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
117 * Store specified sockets on which memory pool to be used by ports
120 uint8_t port_numa[RTE_MAX_ETHPORTS];
123 * Store specified sockets on which RX ring to be used by ports
126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
129 * Store specified sockets on which TX ring to be used by ports
132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
135 * Record the Ethernet address of peer target ports to which packets are
137 * Must be instantiated with the ethernet addresses of peer traffic generator
140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
141 portid_t nb_peer_eth_addrs = 0;
144 * Probed Target Environment.
146 struct rte_port *ports; /**< For all probed ethernet ports. */
147 portid_t nb_ports; /**< Number of probed ethernet ports. */
148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */
151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
154 * Test Forwarding Configuration.
155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports
158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
160 portid_t nb_cfg_ports; /**< Number of configured ports. */
161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */
163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */
166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */
170 * Forwarding engines.
172 struct fwd_engine * fwd_engines[] = {
182 &five_tuple_swap_fwd_engine,
183 #ifdef RTE_LIBRTE_IEEE1588
184 &ieee1588_fwd_engine,
189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
190 uint16_t mempool_flags;
192 struct fwd_config cur_fwd_config;
193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
194 uint32_t retry_enabled;
195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
200 DEFAULT_MBUF_DATA_SIZE
201 }; /**< Mbuf data space size. */
202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if
203 * specified on command-line. */
204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
207 * In container, it cannot terminate the process which running with 'stats-period'
208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
213 * Configuration of packet segments used to scatter received packets
214 * if some of split features is configured.
216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */
218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */
222 * Configuration of packet segments used by the "txonly" processing engine.
224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
226 TXONLY_DEF_PACKET_LEN,
228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
231 /**< Split policy for packets to TX. */
233 uint8_t txonly_multi_flow;
234 /**< Whether multiple flows are generated in TXONLY mode. */
236 uint32_t tx_pkt_times_inter;
237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
239 uint32_t tx_pkt_times_intra;
240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
246 /* current configuration is in DCB or not,0 means it is not in DCB mode */
247 uint8_t dcb_config = 0;
249 /* Whether the dcb is in testing status */
250 uint8_t dcb_test = 0;
253 * Configurable number of RX/TX queues.
255 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
256 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
257 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
260 * Configurable number of RX/TX ring descriptors.
261 * Defaults are supplied by drivers via ethdev.
263 #define RTE_TEST_RX_DESC_DEFAULT 0
264 #define RTE_TEST_TX_DESC_DEFAULT 0
265 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
266 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
268 #define RTE_PMD_PARAM_UNSET -1
270 * Configurable values of RX and TX ring threshold registers.
273 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
274 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
275 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
277 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
278 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
279 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
282 * Configurable value of RX free threshold.
284 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
287 * Configurable value of RX drop enable.
289 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
292 * Configurable value of TX free threshold.
294 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
297 * Configurable value of TX RS bit threshold.
299 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
302 * Configurable value of buffered packets before sending.
304 uint16_t noisy_tx_sw_bufsz;
307 * Configurable value of packet buffer timeout.
309 uint16_t noisy_tx_sw_buf_flush_time;
312 * Configurable value for size of VNF internal memory area
313 * used for simulating noisy neighbour behaviour
315 uint64_t noisy_lkup_mem_sz;
318 * Configurable value of number of random writes done in
319 * VNF simulation memory area.
321 uint64_t noisy_lkup_num_writes;
324 * Configurable value of number of random reads done in
325 * VNF simulation memory area.
327 uint64_t noisy_lkup_num_reads;
330 * Configurable value of number of random reads/writes done in
331 * VNF simulation memory area.
333 uint64_t noisy_lkup_num_reads_writes;
336 * Receive Side Scaling (RSS) configuration.
338 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
341 * Port topology configuration
343 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
346 * Avoids to flush all the RX streams before starts forwarding.
348 uint8_t no_flush_rx = 0; /* flush by default */
351 * Flow API isolated mode.
353 uint8_t flow_isolate_all;
356 * Avoids to check link status when starting/stopping a port.
358 uint8_t no_link_check = 0; /* check by default */
361 * Don't automatically start all ports in interactive mode.
363 uint8_t no_device_start = 0;
366 * Enable link status change notification
368 uint8_t lsc_interrupt = 1; /* enabled by default */
371 * Enable device removal notification.
373 uint8_t rmv_interrupt = 1; /* enabled by default */
375 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
377 /* After attach, port setup is called on event or by iterator */
378 bool setup_on_probe_event = true;
380 /* Clear ptypes on port initialization. */
381 uint8_t clear_ptypes = true;
383 /* Hairpin ports configuration mode. */
384 uint16_t hairpin_mode;
386 /* Pretty printing of ethdev events */
387 static const char * const eth_event_desc[] = {
388 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
389 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
390 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
391 [RTE_ETH_EVENT_INTR_RESET] = "reset",
392 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
393 [RTE_ETH_EVENT_IPSEC] = "IPsec",
394 [RTE_ETH_EVENT_MACSEC] = "MACsec",
395 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
396 [RTE_ETH_EVENT_NEW] = "device probed",
397 [RTE_ETH_EVENT_DESTROY] = "device released",
398 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
399 [RTE_ETH_EVENT_MAX] = NULL,
403 * Display or mask ether events
404 * Default to all events except VF_MBOX
406 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
407 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
408 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
410 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
411 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
412 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
413 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
415 * Decide if all memory are locked for performance.
420 * NIC bypass mode configuration options.
423 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
424 /* The NIC bypass watchdog timeout. */
425 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
429 #ifdef RTE_LIB_LATENCYSTATS
432 * Set when latency stats is enabled in the commandline
434 uint8_t latencystats_enabled;
437 * Lcore ID to serive latency statistics.
439 lcoreid_t latencystats_lcore_id = -1;
444 * Ethernet device configuration.
446 struct rte_eth_rxmode rx_mode = {
447 /* Default maximum frame length.
448 * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
454 struct rte_eth_txmode tx_mode = {
455 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
458 struct rte_fdir_conf fdir_conf = {
459 .mode = RTE_FDIR_MODE_NONE,
460 .pballoc = RTE_FDIR_PBALLOC_64K,
461 .status = RTE_FDIR_REPORT_STATUS,
463 .vlan_tci_mask = 0xFFEF,
465 .src_ip = 0xFFFFFFFF,
466 .dst_ip = 0xFFFFFFFF,
469 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
470 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
472 .src_port_mask = 0xFFFF,
473 .dst_port_mask = 0xFFFF,
474 .mac_addr_byte_mask = 0xFF,
475 .tunnel_type_mask = 1,
476 .tunnel_id_mask = 0xFFFFFFFF,
481 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
484 * Display zero values by default for xstats
486 uint8_t xstats_hide_zero;
489 * Measure of CPU cycles disabled by default
491 uint8_t record_core_cycles;
494 * Display of RX and TX bursts disabled by default
496 uint8_t record_burst_stats;
498 unsigned int num_sockets = 0;
499 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
501 #ifdef RTE_LIB_BITRATESTATS
502 /* Bitrate statistics */
503 struct rte_stats_bitrates *bitrate_data;
504 lcoreid_t bitrate_lcore_id;
505 uint8_t bitrate_enabled;
508 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
509 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
512 * hexadecimal bitmask of RX mq mode can be enabled.
514 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
517 * Used to set forced link speed
519 uint32_t eth_link_speed;
521 /* Forward function declarations */
522 static void setup_attached_port(portid_t pi);
523 static void check_all_ports_link_status(uint32_t port_mask);
524 static int eth_event_callback(portid_t port_id,
525 enum rte_eth_event_type type,
526 void *param, void *ret_param);
527 static void dev_event_callback(const char *device_name,
528 enum rte_dev_event_type type,
532 * Check if all the ports are started.
533 * If yes, return positive value. If not, return zero.
535 static int all_ports_started(void);
537 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
538 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
540 /* Holds the registered mbuf dynamic flags names. */
541 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
544 * Helper function to check if socket is already discovered.
545 * If yes, return positive value. If not, return zero.
548 new_socket_id(unsigned int socket_id)
552 for (i = 0; i < num_sockets; i++) {
553 if (socket_ids[i] == socket_id)
560 * Setup default configuration.
563 set_default_fwd_lcores_config(void)
567 unsigned int sock_num;
570 for (i = 0; i < RTE_MAX_LCORE; i++) {
571 if (!rte_lcore_is_enabled(i))
573 sock_num = rte_lcore_to_socket_id(i);
574 if (new_socket_id(sock_num)) {
575 if (num_sockets >= RTE_MAX_NUMA_NODES) {
576 rte_exit(EXIT_FAILURE,
577 "Total sockets greater than %u\n",
580 socket_ids[num_sockets++] = sock_num;
582 if (i == rte_get_main_lcore())
584 fwd_lcores_cpuids[nb_lc++] = i;
586 nb_lcores = (lcoreid_t) nb_lc;
587 nb_cfg_lcores = nb_lcores;
592 set_def_peer_eth_addrs(void)
596 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
597 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
598 peer_eth_addrs[i].addr_bytes[5] = i;
603 set_default_fwd_ports_config(void)
608 RTE_ETH_FOREACH_DEV(pt_id) {
609 fwd_ports_ids[i++] = pt_id;
611 /* Update sockets info according to the attached device */
612 int socket_id = rte_eth_dev_socket_id(pt_id);
613 if (socket_id >= 0 && new_socket_id(socket_id)) {
614 if (num_sockets >= RTE_MAX_NUMA_NODES) {
615 rte_exit(EXIT_FAILURE,
616 "Total sockets greater than %u\n",
619 socket_ids[num_sockets++] = socket_id;
623 nb_cfg_ports = nb_ports;
624 nb_fwd_ports = nb_ports;
628 set_def_fwd_config(void)
630 set_default_fwd_lcores_config();
631 set_def_peer_eth_addrs();
632 set_default_fwd_ports_config();
635 /* extremely pessimistic estimation of memory required to create a mempool */
637 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
639 unsigned int n_pages, mbuf_per_pg, leftover;
640 uint64_t total_mem, mbuf_mem, obj_sz;
642 /* there is no good way to predict how much space the mempool will
643 * occupy because it will allocate chunks on the fly, and some of those
644 * will come from default DPDK memory while some will come from our
645 * external memory, so just assume 128MB will be enough for everyone.
647 uint64_t hdr_mem = 128 << 20;
649 /* account for possible non-contiguousness */
650 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
652 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
656 mbuf_per_pg = pgsz / obj_sz;
657 leftover = (nb_mbufs % mbuf_per_pg) > 0;
658 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
660 mbuf_mem = n_pages * pgsz;
662 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
664 if (total_mem > SIZE_MAX) {
665 TESTPMD_LOG(ERR, "Memory size too big\n");
668 *out = (size_t)total_mem;
674 pagesz_flags(uint64_t page_sz)
676 /* as per mmap() manpage, all page sizes are log2 of page size
677 * shifted by MAP_HUGE_SHIFT
679 int log2 = rte_log2_u64(page_sz);
681 return (log2 << HUGE_SHIFT);
685 alloc_mem(size_t memsz, size_t pgsz, bool huge)
690 /* allocate anonymous hugepages */
691 flags = MAP_ANONYMOUS | MAP_PRIVATE;
693 flags |= HUGE_FLAG | pagesz_flags(pgsz);
695 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
696 if (addr == MAP_FAILED)
702 struct extmem_param {
706 rte_iova_t *iova_table;
707 unsigned int iova_table_len;
711 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
714 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
715 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */
716 unsigned int cur_page, n_pages, pgsz_idx;
717 size_t mem_sz, cur_pgsz;
718 rte_iova_t *iovas = NULL;
722 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
723 /* skip anything that is too big */
724 if (pgsizes[pgsz_idx] > SIZE_MAX)
727 cur_pgsz = pgsizes[pgsz_idx];
729 /* if we were told not to allocate hugepages, override */
731 cur_pgsz = sysconf(_SC_PAGESIZE);
733 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
735 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
739 /* allocate our memory */
740 addr = alloc_mem(mem_sz, cur_pgsz, huge);
742 /* if we couldn't allocate memory with a specified page size,
743 * that doesn't mean we can't do it with other page sizes, so
749 /* store IOVA addresses for every page in this memory area */
750 n_pages = mem_sz / cur_pgsz;
752 iovas = malloc(sizeof(*iovas) * n_pages);
755 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
758 /* lock memory if it's not huge pages */
762 /* populate IOVA addresses */
763 for (cur_page = 0; cur_page < n_pages; cur_page++) {
768 offset = cur_pgsz * cur_page;
769 cur = RTE_PTR_ADD(addr, offset);
771 /* touch the page before getting its IOVA */
772 *(volatile char *)cur = 0;
774 iova = rte_mem_virt2iova(cur);
776 iovas[cur_page] = iova;
781 /* if we couldn't allocate anything */
787 param->pgsz = cur_pgsz;
788 param->iova_table = iovas;
789 param->iova_table_len = n_pages;
796 munmap(addr, mem_sz);
802 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
804 struct extmem_param param;
807 memset(¶m, 0, sizeof(param));
809 /* check if our heap exists */
810 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
812 /* create our heap */
813 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
815 TESTPMD_LOG(ERR, "Cannot create heap\n");
820 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
822 TESTPMD_LOG(ERR, "Cannot create memory area\n");
826 /* we now have a valid memory area, so add it to heap */
827 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
828 param.addr, param.len, param.iova_table,
829 param.iova_table_len, param.pgsz);
831 /* when using VFIO, memory is automatically mapped for DMA by EAL */
833 /* not needed any more */
834 free(param.iova_table);
837 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
838 munmap(param.addr, param.len);
844 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
850 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
851 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
856 RTE_ETH_FOREACH_DEV(pid) {
857 struct rte_eth_dev *dev =
858 &rte_eth_devices[pid];
860 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
864 "unable to DMA unmap addr 0x%p "
866 memhdr->addr, dev->data->name);
869 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
872 "unable to un-register addr 0x%p\n", memhdr->addr);
877 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
878 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
881 size_t page_size = sysconf(_SC_PAGESIZE);
884 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
888 "unable to register addr 0x%p\n", memhdr->addr);
891 RTE_ETH_FOREACH_DEV(pid) {
892 struct rte_eth_dev *dev =
893 &rte_eth_devices[pid];
895 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
899 "unable to DMA map addr 0x%p "
901 memhdr->addr, dev->data->name);
907 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
908 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
910 struct rte_pktmbuf_extmem *xmem;
911 unsigned int ext_num, zone_num, elt_num;
914 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
915 elt_num = EXTBUF_ZONE_SIZE / elt_size;
916 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
918 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
920 TESTPMD_LOG(ERR, "Cannot allocate memory for "
921 "external buffer descriptors\n");
925 for (ext_num = 0; ext_num < zone_num; ext_num++) {
926 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
927 const struct rte_memzone *mz;
928 char mz_name[RTE_MEMZONE_NAMESIZE];
931 ret = snprintf(mz_name, sizeof(mz_name),
932 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
933 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
934 errno = ENAMETOOLONG;
938 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
940 RTE_MEMZONE_IOVA_CONTIG |
942 RTE_MEMZONE_SIZE_HINT_ONLY,
946 * The caller exits on external buffer creation
947 * error, so there is no need to free memzones.
953 xseg->buf_ptr = mz->addr;
954 xseg->buf_iova = mz->iova;
955 xseg->buf_len = EXTBUF_ZONE_SIZE;
956 xseg->elt_size = elt_size;
958 if (ext_num == 0 && xmem != NULL) {
967 * Configuration initialisation done once at init time.
969 static struct rte_mempool *
970 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
971 unsigned int socket_id, uint16_t size_idx)
973 char pool_name[RTE_MEMPOOL_NAMESIZE];
974 struct rte_mempool *rte_mp = NULL;
977 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
978 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
981 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
982 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
984 switch (mp_alloc_type) {
985 case MP_ALLOC_NATIVE:
987 /* wrapper to rte_mempool_create() */
988 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
989 rte_mbuf_best_mempool_ops());
990 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
991 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
996 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
997 mb_size, (unsigned int) mb_mempool_cache,
998 sizeof(struct rte_pktmbuf_pool_private),
999 socket_id, mempool_flags);
1003 if (rte_mempool_populate_anon(rte_mp) == 0) {
1004 rte_mempool_free(rte_mp);
1008 rte_pktmbuf_pool_init(rte_mp, NULL);
1009 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1010 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1014 case MP_ALLOC_XMEM_HUGE:
1017 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1019 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1020 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1023 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1024 if (heap_socket < 0)
1025 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1027 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1028 rte_mbuf_best_mempool_ops());
1029 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1030 mb_mempool_cache, 0, mbuf_seg_size,
1036 struct rte_pktmbuf_extmem *ext_mem;
1037 unsigned int ext_num;
1039 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1040 socket_id, pool_name, &ext_mem);
1042 rte_exit(EXIT_FAILURE,
1043 "Can't create pinned data buffers\n");
1045 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1046 rte_mbuf_best_mempool_ops());
1047 rte_mp = rte_pktmbuf_pool_create_extbuf
1048 (pool_name, nb_mbuf, mb_mempool_cache,
1049 0, mbuf_seg_size, socket_id,
1056 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1061 if (rte_mp == NULL) {
1062 rte_exit(EXIT_FAILURE,
1063 "Creation of mbuf pool for socket %u failed: %s\n",
1064 socket_id, rte_strerror(rte_errno));
1065 } else if (verbose_level > 0) {
1066 rte_mempool_dump(stdout, rte_mp);
1072 * Check given socket id is valid or not with NUMA mode,
1073 * if valid, return 0, else return -1
1076 check_socket_id(const unsigned int socket_id)
1078 static int warning_once = 0;
1080 if (new_socket_id(socket_id)) {
1081 if (!warning_once && numa_support)
1082 printf("Warning: NUMA should be configured manually by"
1083 " using --port-numa-config and"
1084 " --ring-numa-config parameters along with"
1093 * Get the allowed maximum number of RX queues.
1094 * *pid return the port id which has minimal value of
1095 * max_rx_queues in all ports.
1098 get_allowed_max_nb_rxq(portid_t *pid)
1100 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1101 bool max_rxq_valid = false;
1103 struct rte_eth_dev_info dev_info;
1105 RTE_ETH_FOREACH_DEV(pi) {
1106 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1109 max_rxq_valid = true;
1110 if (dev_info.max_rx_queues < allowed_max_rxq) {
1111 allowed_max_rxq = dev_info.max_rx_queues;
1115 return max_rxq_valid ? allowed_max_rxq : 0;
1119 * Check input rxq is valid or not.
1120 * If input rxq is not greater than any of maximum number
1121 * of RX queues of all ports, it is valid.
1122 * if valid, return 0, else return -1
1125 check_nb_rxq(queueid_t rxq)
1127 queueid_t allowed_max_rxq;
1130 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1131 if (rxq > allowed_max_rxq) {
1132 printf("Fail: input rxq (%u) can't be greater "
1133 "than max_rx_queues (%u) of port %u\n",
1143 * Get the allowed maximum number of TX queues.
1144 * *pid return the port id which has minimal value of
1145 * max_tx_queues in all ports.
1148 get_allowed_max_nb_txq(portid_t *pid)
1150 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1151 bool max_txq_valid = false;
1153 struct rte_eth_dev_info dev_info;
1155 RTE_ETH_FOREACH_DEV(pi) {
1156 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1159 max_txq_valid = true;
1160 if (dev_info.max_tx_queues < allowed_max_txq) {
1161 allowed_max_txq = dev_info.max_tx_queues;
1165 return max_txq_valid ? allowed_max_txq : 0;
1169 * Check input txq is valid or not.
1170 * If input txq is not greater than any of maximum number
1171 * of TX queues of all ports, it is valid.
1172 * if valid, return 0, else return -1
1175 check_nb_txq(queueid_t txq)
1177 queueid_t allowed_max_txq;
1180 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1181 if (txq > allowed_max_txq) {
1182 printf("Fail: input txq (%u) can't be greater "
1183 "than max_tx_queues (%u) of port %u\n",
1193 * Get the allowed maximum number of RXDs of every rx queue.
1194 * *pid return the port id which has minimal value of
1195 * max_rxd in all queues of all ports.
1198 get_allowed_max_nb_rxd(portid_t *pid)
1200 uint16_t allowed_max_rxd = UINT16_MAX;
1202 struct rte_eth_dev_info dev_info;
1204 RTE_ETH_FOREACH_DEV(pi) {
1205 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1208 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1209 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1213 return allowed_max_rxd;
1217 * Get the allowed minimal number of RXDs of every rx queue.
1218 * *pid return the port id which has minimal value of
1219 * min_rxd in all queues of all ports.
1222 get_allowed_min_nb_rxd(portid_t *pid)
1224 uint16_t allowed_min_rxd = 0;
1226 struct rte_eth_dev_info dev_info;
1228 RTE_ETH_FOREACH_DEV(pi) {
1229 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1232 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1233 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1238 return allowed_min_rxd;
1242 * Check input rxd is valid or not.
1243 * If input rxd is not greater than any of maximum number
1244 * of RXDs of every Rx queues and is not less than any of
1245 * minimal number of RXDs of every Rx queues, it is valid.
1246 * if valid, return 0, else return -1
1249 check_nb_rxd(queueid_t rxd)
1251 uint16_t allowed_max_rxd;
1252 uint16_t allowed_min_rxd;
1255 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1256 if (rxd > allowed_max_rxd) {
1257 printf("Fail: input rxd (%u) can't be greater "
1258 "than max_rxds (%u) of port %u\n",
1265 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1266 if (rxd < allowed_min_rxd) {
1267 printf("Fail: input rxd (%u) can't be less "
1268 "than min_rxds (%u) of port %u\n",
1279 * Get the allowed maximum number of TXDs of every rx queues.
1280 * *pid return the port id which has minimal value of
1281 * max_txd in every tx queue.
1284 get_allowed_max_nb_txd(portid_t *pid)
1286 uint16_t allowed_max_txd = UINT16_MAX;
1288 struct rte_eth_dev_info dev_info;
1290 RTE_ETH_FOREACH_DEV(pi) {
1291 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1294 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1295 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1299 return allowed_max_txd;
1303 * Get the allowed maximum number of TXDs of every tx queues.
1304 * *pid return the port id which has minimal value of
1305 * min_txd in every tx queue.
1308 get_allowed_min_nb_txd(portid_t *pid)
1310 uint16_t allowed_min_txd = 0;
1312 struct rte_eth_dev_info dev_info;
1314 RTE_ETH_FOREACH_DEV(pi) {
1315 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1318 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1319 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1324 return allowed_min_txd;
1328 * Check input txd is valid or not.
1329 * If input txd is not greater than any of maximum number
1330 * of TXDs of every Rx queues, it is valid.
1331 * if valid, return 0, else return -1
1334 check_nb_txd(queueid_t txd)
1336 uint16_t allowed_max_txd;
1337 uint16_t allowed_min_txd;
1340 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1341 if (txd > allowed_max_txd) {
1342 printf("Fail: input txd (%u) can't be greater "
1343 "than max_txds (%u) of port %u\n",
1350 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1351 if (txd < allowed_min_txd) {
1352 printf("Fail: input txd (%u) can't be less "
1353 "than min_txds (%u) of port %u\n",
1364 * Get the allowed maximum number of hairpin queues.
1365 * *pid return the port id which has minimal value of
1366 * max_hairpin_queues in all ports.
1369 get_allowed_max_nb_hairpinq(portid_t *pid)
1371 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1373 struct rte_eth_hairpin_cap cap;
1375 RTE_ETH_FOREACH_DEV(pi) {
1376 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1380 if (cap.max_nb_queues < allowed_max_hairpinq) {
1381 allowed_max_hairpinq = cap.max_nb_queues;
1385 return allowed_max_hairpinq;
1389 * Check input hairpin is valid or not.
1390 * If input hairpin is not greater than any of maximum number
1391 * of hairpin queues of all ports, it is valid.
1392 * if valid, return 0, else return -1
1395 check_nb_hairpinq(queueid_t hairpinq)
1397 queueid_t allowed_max_hairpinq;
1400 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1401 if (hairpinq > allowed_max_hairpinq) {
1402 printf("Fail: input hairpin (%u) can't be greater "
1403 "than max_hairpin_queues (%u) of port %u\n",
1404 hairpinq, allowed_max_hairpinq, pid);
1414 struct rte_port *port;
1415 struct rte_mempool *mbp;
1416 unsigned int nb_mbuf_per_pool;
1418 uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1419 struct rte_gro_param gro_param;
1426 memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1428 /* Configuration of logical cores. */
1429 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1430 sizeof(struct fwd_lcore *) * nb_lcores,
1431 RTE_CACHE_LINE_SIZE);
1432 if (fwd_lcores == NULL) {
1433 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1434 "failed\n", nb_lcores);
1436 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1437 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1438 sizeof(struct fwd_lcore),
1439 RTE_CACHE_LINE_SIZE);
1440 if (fwd_lcores[lc_id] == NULL) {
1441 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1444 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1447 RTE_ETH_FOREACH_DEV(pid) {
1449 /* Apply default TxRx configuration for all ports */
1450 port->dev_conf.txmode = tx_mode;
1451 port->dev_conf.rxmode = rx_mode;
1453 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1455 rte_exit(EXIT_FAILURE,
1456 "rte_eth_dev_info_get() failed\n");
1458 ret = update_jumbo_frame_offload(pid);
1460 printf("Updating jumbo frame offload failed for port %u\n",
1463 if (!(port->dev_info.tx_offload_capa &
1464 DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1465 port->dev_conf.txmode.offloads &=
1466 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1468 if (port_numa[pid] != NUMA_NO_CONFIG)
1469 port_per_socket[port_numa[pid]]++;
1471 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1474 * if socket_id is invalid,
1475 * set to the first available socket.
1477 if (check_socket_id(socket_id) < 0)
1478 socket_id = socket_ids[0];
1479 port_per_socket[socket_id]++;
1483 /* Apply Rx offloads configuration */
1484 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1485 port->rx_conf[k].offloads =
1486 port->dev_conf.rxmode.offloads;
1487 /* Apply Tx offloads configuration */
1488 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1489 port->tx_conf[k].offloads =
1490 port->dev_conf.txmode.offloads;
1493 port->dev_conf.link_speeds = eth_link_speed;
1495 /* set flag to initialize port/queue */
1496 port->need_reconfig = 1;
1497 port->need_reconfig_queues = 1;
1498 port->tx_metadata = 0;
1500 /* Check for maximum number of segments per MTU. Accordingly
1501 * update the mbuf data size.
1503 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1504 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1505 data_size = rx_mode.max_rx_pkt_len /
1506 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1508 if ((data_size + RTE_PKTMBUF_HEADROOM) >
1509 mbuf_data_size[0]) {
1510 mbuf_data_size[0] = data_size +
1511 RTE_PKTMBUF_HEADROOM;
1518 TESTPMD_LOG(WARNING,
1519 "Configured mbuf size of the first segment %hu\n",
1522 * Create pools of mbuf.
1523 * If NUMA support is disabled, create a single pool of mbuf in
1524 * socket 0 memory by default.
1525 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1527 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1528 * nb_txd can be configured at run time.
1530 if (param_total_num_mbufs)
1531 nb_mbuf_per_pool = param_total_num_mbufs;
1533 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1534 (nb_lcores * mb_mempool_cache) +
1535 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1536 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1542 for (i = 0; i < num_sockets; i++)
1543 for (j = 0; j < mbuf_data_size_n; j++)
1544 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1545 mbuf_pool_create(mbuf_data_size[j],
1551 for (i = 0; i < mbuf_data_size_n; i++)
1552 mempools[i] = mbuf_pool_create
1555 socket_num == UMA_NO_CONFIG ?
1561 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1562 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1564 * Records which Mbuf pool to use by each logical core, if needed.
1566 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1567 mbp = mbuf_pool_find(
1568 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1571 mbp = mbuf_pool_find(0, 0);
1572 fwd_lcores[lc_id]->mbp = mbp;
1573 /* initialize GSO context */
1574 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1575 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1576 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1577 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1579 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1582 /* Configuration of packet forwarding streams. */
1583 if (init_fwd_streams() < 0)
1584 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1588 /* create a gro context for each lcore */
1589 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1590 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1591 gro_param.max_item_per_flow = MAX_PKT_BURST;
1592 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1593 gro_param.socket_id = rte_lcore_to_socket_id(
1594 fwd_lcores_cpuids[lc_id]);
1595 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1596 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1597 rte_exit(EXIT_FAILURE,
1598 "rte_gro_ctx_create() failed\n");
1605 reconfig(portid_t new_port_id, unsigned socket_id)
1607 struct rte_port *port;
1610 /* Reconfiguration of Ethernet ports. */
1611 port = &ports[new_port_id];
1613 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1617 /* set flag to initialize port/queue */
1618 port->need_reconfig = 1;
1619 port->need_reconfig_queues = 1;
1620 port->socket_id = socket_id;
1627 init_fwd_streams(void)
1630 struct rte_port *port;
1631 streamid_t sm_id, nb_fwd_streams_new;
1634 /* set socket id according to numa or not */
1635 RTE_ETH_FOREACH_DEV(pid) {
1637 if (nb_rxq > port->dev_info.max_rx_queues) {
1638 printf("Fail: nb_rxq(%d) is greater than "
1639 "max_rx_queues(%d)\n", nb_rxq,
1640 port->dev_info.max_rx_queues);
1643 if (nb_txq > port->dev_info.max_tx_queues) {
1644 printf("Fail: nb_txq(%d) is greater than "
1645 "max_tx_queues(%d)\n", nb_txq,
1646 port->dev_info.max_tx_queues);
1650 if (port_numa[pid] != NUMA_NO_CONFIG)
1651 port->socket_id = port_numa[pid];
1653 port->socket_id = rte_eth_dev_socket_id(pid);
1656 * if socket_id is invalid,
1657 * set to the first available socket.
1659 if (check_socket_id(port->socket_id) < 0)
1660 port->socket_id = socket_ids[0];
1664 if (socket_num == UMA_NO_CONFIG)
1665 port->socket_id = 0;
1667 port->socket_id = socket_num;
1671 q = RTE_MAX(nb_rxq, nb_txq);
1673 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1676 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1677 if (nb_fwd_streams_new == nb_fwd_streams)
1680 if (fwd_streams != NULL) {
1681 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1682 if (fwd_streams[sm_id] == NULL)
1684 rte_free(fwd_streams[sm_id]);
1685 fwd_streams[sm_id] = NULL;
1687 rte_free(fwd_streams);
1692 nb_fwd_streams = nb_fwd_streams_new;
1693 if (nb_fwd_streams) {
1694 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1695 sizeof(struct fwd_stream *) * nb_fwd_streams,
1696 RTE_CACHE_LINE_SIZE);
1697 if (fwd_streams == NULL)
1698 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1699 " (struct fwd_stream *)) failed\n",
1702 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1703 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1704 " struct fwd_stream", sizeof(struct fwd_stream),
1705 RTE_CACHE_LINE_SIZE);
1706 if (fwd_streams[sm_id] == NULL)
1707 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1708 "(struct fwd_stream) failed\n");
1716 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1718 uint64_t total_burst, sburst;
1720 uint64_t burst_stats[4];
1721 uint16_t pktnb_stats[4];
1723 int burst_percent[4], sburstp;
1727 * First compute the total number of packet bursts and the
1728 * two highest numbers of bursts of the same number of packets.
1730 memset(&burst_stats, 0x0, sizeof(burst_stats));
1731 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1733 /* Show stats for 0 burst size always */
1734 total_burst = pbs->pkt_burst_spread[0];
1735 burst_stats[0] = pbs->pkt_burst_spread[0];
1738 /* Find the next 2 burst sizes with highest occurrences. */
1739 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1740 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1745 total_burst += nb_burst;
1747 if (nb_burst > burst_stats[1]) {
1748 burst_stats[2] = burst_stats[1];
1749 pktnb_stats[2] = pktnb_stats[1];
1750 burst_stats[1] = nb_burst;
1751 pktnb_stats[1] = nb_pkt;
1752 } else if (nb_burst > burst_stats[2]) {
1753 burst_stats[2] = nb_burst;
1754 pktnb_stats[2] = nb_pkt;
1757 if (total_burst == 0)
1760 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1761 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1763 printf("%d%% of other]\n", 100 - sburstp);
1767 sburst += burst_stats[i];
1768 if (sburst == total_burst) {
1769 printf("%d%% of %d pkts]\n",
1770 100 - sburstp, (int) pktnb_stats[i]);
1775 (double)burst_stats[i] / total_burst * 100;
1776 printf("%d%% of %d pkts + ",
1777 burst_percent[i], (int) pktnb_stats[i]);
1778 sburstp += burst_percent[i];
1783 fwd_stream_stats_display(streamid_t stream_id)
1785 struct fwd_stream *fs;
1786 static const char *fwd_top_stats_border = "-------";
1788 fs = fwd_streams[stream_id];
1789 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1790 (fs->fwd_dropped == 0))
1792 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1793 "TX Port=%2d/Queue=%2d %s\n",
1794 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1795 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1796 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1797 " TX-dropped: %-14"PRIu64,
1798 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1800 /* if checksum mode */
1801 if (cur_fwd_eng == &csum_fwd_engine) {
1802 printf(" RX- bad IP checksum: %-14"PRIu64
1803 " Rx- bad L4 checksum: %-14"PRIu64
1804 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1805 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1806 fs->rx_bad_outer_l4_csum);
1807 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1808 fs->rx_bad_outer_ip_csum);
1813 if (record_burst_stats) {
1814 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1815 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1820 fwd_stats_display(void)
1822 static const char *fwd_stats_border = "----------------------";
1823 static const char *acc_stats_border = "+++++++++++++++";
1825 struct fwd_stream *rx_stream;
1826 struct fwd_stream *tx_stream;
1827 uint64_t tx_dropped;
1828 uint64_t rx_bad_ip_csum;
1829 uint64_t rx_bad_l4_csum;
1830 uint64_t rx_bad_outer_l4_csum;
1831 uint64_t rx_bad_outer_ip_csum;
1832 } ports_stats[RTE_MAX_ETHPORTS];
1833 uint64_t total_rx_dropped = 0;
1834 uint64_t total_tx_dropped = 0;
1835 uint64_t total_rx_nombuf = 0;
1836 struct rte_eth_stats stats;
1837 uint64_t fwd_cycles = 0;
1838 uint64_t total_recv = 0;
1839 uint64_t total_xmit = 0;
1840 struct rte_port *port;
1845 memset(ports_stats, 0, sizeof(ports_stats));
1847 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1848 struct fwd_stream *fs = fwd_streams[sm_id];
1850 if (cur_fwd_config.nb_fwd_streams >
1851 cur_fwd_config.nb_fwd_ports) {
1852 fwd_stream_stats_display(sm_id);
1854 ports_stats[fs->tx_port].tx_stream = fs;
1855 ports_stats[fs->rx_port].rx_stream = fs;
1858 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1860 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1861 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1862 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1863 fs->rx_bad_outer_l4_csum;
1864 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1865 fs->rx_bad_outer_ip_csum;
1867 if (record_core_cycles)
1868 fwd_cycles += fs->core_cycles;
1870 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1871 pt_id = fwd_ports_ids[i];
1872 port = &ports[pt_id];
1874 rte_eth_stats_get(pt_id, &stats);
1875 stats.ipackets -= port->stats.ipackets;
1876 stats.opackets -= port->stats.opackets;
1877 stats.ibytes -= port->stats.ibytes;
1878 stats.obytes -= port->stats.obytes;
1879 stats.imissed -= port->stats.imissed;
1880 stats.oerrors -= port->stats.oerrors;
1881 stats.rx_nombuf -= port->stats.rx_nombuf;
1883 total_recv += stats.ipackets;
1884 total_xmit += stats.opackets;
1885 total_rx_dropped += stats.imissed;
1886 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1887 total_tx_dropped += stats.oerrors;
1888 total_rx_nombuf += stats.rx_nombuf;
1890 printf("\n %s Forward statistics for port %-2d %s\n",
1891 fwd_stats_border, pt_id, fwd_stats_border);
1893 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1894 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1895 stats.ipackets + stats.imissed);
1897 if (cur_fwd_eng == &csum_fwd_engine) {
1898 printf(" Bad-ipcsum: %-14"PRIu64
1899 " Bad-l4csum: %-14"PRIu64
1900 "Bad-outer-l4csum: %-14"PRIu64"\n",
1901 ports_stats[pt_id].rx_bad_ip_csum,
1902 ports_stats[pt_id].rx_bad_l4_csum,
1903 ports_stats[pt_id].rx_bad_outer_l4_csum);
1904 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n",
1905 ports_stats[pt_id].rx_bad_outer_ip_csum);
1907 if (stats.ierrors + stats.rx_nombuf > 0) {
1908 printf(" RX-error: %-"PRIu64"\n", stats.ierrors);
1909 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1912 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1913 "TX-total: %-"PRIu64"\n",
1914 stats.opackets, ports_stats[pt_id].tx_dropped,
1915 stats.opackets + ports_stats[pt_id].tx_dropped);
1917 if (record_burst_stats) {
1918 if (ports_stats[pt_id].rx_stream)
1919 pkt_burst_stats_display("RX",
1920 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1921 if (ports_stats[pt_id].tx_stream)
1922 pkt_burst_stats_display("TX",
1923 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1926 printf(" %s--------------------------------%s\n",
1927 fwd_stats_border, fwd_stats_border);
1930 printf("\n %s Accumulated forward statistics for all ports"
1932 acc_stats_border, acc_stats_border);
1933 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1935 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1937 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1938 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1939 if (total_rx_nombuf > 0)
1940 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1941 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
1943 acc_stats_border, acc_stats_border);
1944 if (record_core_cycles) {
1945 #define CYC_PER_MHZ 1E6
1946 if (total_recv > 0 || total_xmit > 0) {
1947 uint64_t total_pkts = 0;
1948 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1949 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1950 total_pkts = total_xmit;
1952 total_pkts = total_recv;
1954 printf("\n CPU cycles/packet=%.2F (total cycles="
1955 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1957 (double) fwd_cycles / total_pkts,
1958 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1959 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1965 fwd_stats_reset(void)
1971 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1972 pt_id = fwd_ports_ids[i];
1973 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1975 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1976 struct fwd_stream *fs = fwd_streams[sm_id];
1980 fs->fwd_dropped = 0;
1981 fs->rx_bad_ip_csum = 0;
1982 fs->rx_bad_l4_csum = 0;
1983 fs->rx_bad_outer_l4_csum = 0;
1984 fs->rx_bad_outer_ip_csum = 0;
1986 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1987 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1988 fs->core_cycles = 0;
1993 flush_fwd_rx_queues(void)
1995 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2002 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2003 uint64_t timer_period;
2005 /* convert to number of cycles */
2006 timer_period = rte_get_timer_hz(); /* 1 second timeout */
2008 for (j = 0; j < 2; j++) {
2009 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2010 for (rxq = 0; rxq < nb_rxq; rxq++) {
2011 port_id = fwd_ports_ids[rxp];
2013 * testpmd can stuck in the below do while loop
2014 * if rte_eth_rx_burst() always returns nonzero
2015 * packets. So timer is added to exit this loop
2016 * after 1sec timer expiry.
2018 prev_tsc = rte_rdtsc();
2020 nb_rx = rte_eth_rx_burst(port_id, rxq,
2021 pkts_burst, MAX_PKT_BURST);
2022 for (i = 0; i < nb_rx; i++)
2023 rte_pktmbuf_free(pkts_burst[i]);
2025 cur_tsc = rte_rdtsc();
2026 diff_tsc = cur_tsc - prev_tsc;
2027 timer_tsc += diff_tsc;
2028 } while ((nb_rx > 0) &&
2029 (timer_tsc < timer_period));
2033 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2038 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2040 struct fwd_stream **fsm;
2043 #ifdef RTE_LIB_BITRATESTATS
2044 uint64_t tics_per_1sec;
2045 uint64_t tics_datum;
2046 uint64_t tics_current;
2047 uint16_t i, cnt_ports;
2049 cnt_ports = nb_ports;
2050 tics_datum = rte_rdtsc();
2051 tics_per_1sec = rte_get_timer_hz();
2053 fsm = &fwd_streams[fc->stream_idx];
2054 nb_fs = fc->stream_nb;
2056 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2057 (*pkt_fwd)(fsm[sm_id]);
2058 #ifdef RTE_LIB_BITRATESTATS
2059 if (bitrate_enabled != 0 &&
2060 bitrate_lcore_id == rte_lcore_id()) {
2061 tics_current = rte_rdtsc();
2062 if (tics_current - tics_datum >= tics_per_1sec) {
2063 /* Periodic bitrate calculation */
2064 for (i = 0; i < cnt_ports; i++)
2065 rte_stats_bitrate_calc(bitrate_data,
2067 tics_datum = tics_current;
2071 #ifdef RTE_LIB_LATENCYSTATS
2072 if (latencystats_enabled != 0 &&
2073 latencystats_lcore_id == rte_lcore_id())
2074 rte_latencystats_update();
2077 } while (! fc->stopped);
2081 start_pkt_forward_on_core(void *fwd_arg)
2083 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2084 cur_fwd_config.fwd_eng->packet_fwd);
2089 * Run the TXONLY packet forwarding engine to send a single burst of packets.
2090 * Used to start communication flows in network loopback test configurations.
2093 run_one_txonly_burst_on_core(void *fwd_arg)
2095 struct fwd_lcore *fwd_lc;
2096 struct fwd_lcore tmp_lcore;
2098 fwd_lc = (struct fwd_lcore *) fwd_arg;
2099 tmp_lcore = *fwd_lc;
2100 tmp_lcore.stopped = 1;
2101 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2106 * Launch packet forwarding:
2107 * - Setup per-port forwarding context.
2108 * - launch logical cores with their forwarding configuration.
2111 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2113 port_fwd_begin_t port_fwd_begin;
2118 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2119 if (port_fwd_begin != NULL) {
2120 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2121 (*port_fwd_begin)(fwd_ports_ids[i]);
2123 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2124 lc_id = fwd_lcores_cpuids[i];
2125 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2126 fwd_lcores[i]->stopped = 0;
2127 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2128 fwd_lcores[i], lc_id);
2130 printf("launch lcore %u failed - diag=%d\n",
2137 * Launch packet forwarding configuration.
2140 start_packet_forwarding(int with_tx_first)
2142 port_fwd_begin_t port_fwd_begin;
2143 port_fwd_end_t port_fwd_end;
2144 struct rte_port *port;
2148 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2149 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2151 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2152 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2154 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2155 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2156 (!nb_rxq || !nb_txq))
2157 rte_exit(EXIT_FAILURE,
2158 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2159 cur_fwd_eng->fwd_mode_name);
2161 if (all_ports_started() == 0) {
2162 printf("Not all ports were started\n");
2165 if (test_done == 0) {
2166 printf("Packet forwarding already started\n");
2172 for (i = 0; i < nb_fwd_ports; i++) {
2173 pt_id = fwd_ports_ids[i];
2174 port = &ports[pt_id];
2175 if (!port->dcb_flag) {
2176 printf("In DCB mode, all forwarding ports must "
2177 "be configured in this mode.\n");
2181 if (nb_fwd_lcores == 1) {
2182 printf("In DCB mode,the nb forwarding cores "
2183 "should be larger than 1.\n");
2192 flush_fwd_rx_queues();
2194 pkt_fwd_config_display(&cur_fwd_config);
2195 rxtx_config_display();
2198 if (with_tx_first) {
2199 port_fwd_begin = tx_only_engine.port_fwd_begin;
2200 if (port_fwd_begin != NULL) {
2201 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2202 (*port_fwd_begin)(fwd_ports_ids[i]);
2204 while (with_tx_first--) {
2205 launch_packet_forwarding(
2206 run_one_txonly_burst_on_core);
2207 rte_eal_mp_wait_lcore();
2209 port_fwd_end = tx_only_engine.port_fwd_end;
2210 if (port_fwd_end != NULL) {
2211 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2212 (*port_fwd_end)(fwd_ports_ids[i]);
2215 launch_packet_forwarding(start_pkt_forward_on_core);
2219 stop_packet_forwarding(void)
2221 port_fwd_end_t port_fwd_end;
2227 printf("Packet forwarding not started\n");
2230 printf("Telling cores to stop...");
2231 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2232 fwd_lcores[lc_id]->stopped = 1;
2233 printf("\nWaiting for lcores to finish...\n");
2234 rte_eal_mp_wait_lcore();
2235 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2236 if (port_fwd_end != NULL) {
2237 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2238 pt_id = fwd_ports_ids[i];
2239 (*port_fwd_end)(pt_id);
2243 fwd_stats_display();
2245 printf("\nDone.\n");
2250 dev_set_link_up(portid_t pid)
2252 if (rte_eth_dev_set_link_up(pid) < 0)
2253 printf("\nSet link up fail.\n");
2257 dev_set_link_down(portid_t pid)
2259 if (rte_eth_dev_set_link_down(pid) < 0)
2260 printf("\nSet link down fail.\n");
2264 all_ports_started(void)
2267 struct rte_port *port;
2269 RTE_ETH_FOREACH_DEV(pi) {
2271 /* Check if there is a port which is not started */
2272 if ((port->port_status != RTE_PORT_STARTED) &&
2273 (port->slave_flag == 0))
2277 /* No port is not started */
2282 port_is_stopped(portid_t port_id)
2284 struct rte_port *port = &ports[port_id];
2286 if ((port->port_status != RTE_PORT_STOPPED) &&
2287 (port->slave_flag == 0))
2293 all_ports_stopped(void)
2297 RTE_ETH_FOREACH_DEV(pi) {
2298 if (!port_is_stopped(pi))
2306 port_is_started(portid_t port_id)
2308 if (port_id_is_invalid(port_id, ENABLED_WARN))
2311 if (ports[port_id].port_status != RTE_PORT_STARTED)
2317 /* Configure the Rx and Tx hairpin queues for the selected port. */
2319 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2322 struct rte_eth_hairpin_conf hairpin_conf = {
2327 struct rte_port *port = &ports[pi];
2328 uint16_t peer_rx_port = pi;
2329 uint16_t peer_tx_port = pi;
2330 uint32_t manual = 1;
2331 uint32_t tx_exp = hairpin_mode & 0x10;
2333 if (!(hairpin_mode & 0xf)) {
2337 } else if (hairpin_mode & 0x1) {
2338 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2339 RTE_ETH_DEV_NO_OWNER);
2340 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2341 peer_tx_port = rte_eth_find_next_owned_by(0,
2342 RTE_ETH_DEV_NO_OWNER);
2343 if (p_pi != RTE_MAX_ETHPORTS) {
2344 peer_rx_port = p_pi;
2348 /* Last port will be the peer RX port of the first. */
2349 RTE_ETH_FOREACH_DEV(next_pi)
2350 peer_rx_port = next_pi;
2353 } else if (hairpin_mode & 0x2) {
2355 peer_rx_port = p_pi;
2357 peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2358 RTE_ETH_DEV_NO_OWNER);
2359 if (peer_rx_port >= RTE_MAX_ETHPORTS)
2362 peer_tx_port = peer_rx_port;
2366 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2367 hairpin_conf.peers[0].port = peer_rx_port;
2368 hairpin_conf.peers[0].queue = i + nb_rxq;
2369 hairpin_conf.manual_bind = !!manual;
2370 hairpin_conf.tx_explicit = !!tx_exp;
2371 diag = rte_eth_tx_hairpin_queue_setup
2372 (pi, qi, nb_txd, &hairpin_conf);
2377 /* Fail to setup rx queue, return */
2378 if (rte_atomic16_cmpset(&(port->port_status),
2380 RTE_PORT_STOPPED) == 0)
2381 printf("Port %d can not be set back "
2382 "to stopped\n", pi);
2383 printf("Fail to configure port %d hairpin "
2385 /* try to reconfigure queues next time */
2386 port->need_reconfig_queues = 1;
2389 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2390 hairpin_conf.peers[0].port = peer_tx_port;
2391 hairpin_conf.peers[0].queue = i + nb_txq;
2392 hairpin_conf.manual_bind = !!manual;
2393 hairpin_conf.tx_explicit = !!tx_exp;
2394 diag = rte_eth_rx_hairpin_queue_setup
2395 (pi, qi, nb_rxd, &hairpin_conf);
2400 /* Fail to setup rx queue, return */
2401 if (rte_atomic16_cmpset(&(port->port_status),
2403 RTE_PORT_STOPPED) == 0)
2404 printf("Port %d can not be set back "
2405 "to stopped\n", pi);
2406 printf("Fail to configure port %d hairpin "
2408 /* try to reconfigure queues next time */
2409 port->need_reconfig_queues = 1;
2415 /* Configure the Rx with optional split. */
2417 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2418 uint16_t nb_rx_desc, unsigned int socket_id,
2419 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2421 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2422 unsigned int i, mp_n;
2425 if (rx_pkt_nb_segs <= 1 ||
2426 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2427 rx_conf->rx_seg = NULL;
2428 rx_conf->rx_nseg = 0;
2429 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2430 nb_rx_desc, socket_id,
2434 for (i = 0; i < rx_pkt_nb_segs; i++) {
2435 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2436 struct rte_mempool *mpx;
2438 * Use last valid pool for the segments with number
2439 * exceeding the pool index.
2441 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2442 mpx = mbuf_pool_find(socket_id, mp_n);
2443 /* Handle zero as mbuf data buffer size. */
2444 rx_seg->length = rx_pkt_seg_lengths[i] ?
2445 rx_pkt_seg_lengths[i] :
2446 mbuf_data_size[mp_n];
2447 rx_seg->offset = i < rx_pkt_nb_offs ?
2448 rx_pkt_seg_offsets[i] : 0;
2449 rx_seg->mp = mpx ? mpx : mp;
2451 rx_conf->rx_nseg = rx_pkt_nb_segs;
2452 rx_conf->rx_seg = rx_useg;
2453 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2454 socket_id, rx_conf, NULL);
2455 rx_conf->rx_seg = NULL;
2456 rx_conf->rx_nseg = 0;
2461 start_port(portid_t pid)
2463 int diag, need_check_link_status = -1;
2465 portid_t p_pi = RTE_MAX_ETHPORTS;
2466 portid_t pl[RTE_MAX_ETHPORTS];
2467 portid_t peer_pl[RTE_MAX_ETHPORTS];
2468 uint16_t cnt_pi = 0;
2469 uint16_t cfg_pi = 0;
2472 struct rte_port *port;
2473 struct rte_ether_addr mac_addr;
2474 struct rte_eth_hairpin_cap cap;
2476 if (port_id_is_invalid(pid, ENABLED_WARN))
2481 RTE_ETH_FOREACH_DEV(pi) {
2482 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2485 need_check_link_status = 0;
2487 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2488 RTE_PORT_HANDLING) == 0) {
2489 printf("Port %d is now not stopped\n", pi);
2493 if (port->need_reconfig > 0) {
2494 port->need_reconfig = 0;
2496 if (flow_isolate_all) {
2497 int ret = port_flow_isolate(pi, 1);
2499 printf("Failed to apply isolated"
2500 " mode on port %d\n", pi);
2504 configure_rxtx_dump_callbacks(0);
2505 printf("Configuring Port %d (socket %u)\n", pi,
2507 if (nb_hairpinq > 0 &&
2508 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2509 printf("Port %d doesn't support hairpin "
2513 /* configure port */
2514 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2515 nb_txq + nb_hairpinq,
2518 if (rte_atomic16_cmpset(&(port->port_status),
2519 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2520 printf("Port %d can not be set back "
2521 "to stopped\n", pi);
2522 printf("Fail to configure port %d\n", pi);
2523 /* try to reconfigure port next time */
2524 port->need_reconfig = 1;
2528 if (port->need_reconfig_queues > 0) {
2529 port->need_reconfig_queues = 0;
2530 /* setup tx queues */
2531 for (qi = 0; qi < nb_txq; qi++) {
2532 if ((numa_support) &&
2533 (txring_numa[pi] != NUMA_NO_CONFIG))
2534 diag = rte_eth_tx_queue_setup(pi, qi,
2535 port->nb_tx_desc[qi],
2537 &(port->tx_conf[qi]));
2539 diag = rte_eth_tx_queue_setup(pi, qi,
2540 port->nb_tx_desc[qi],
2542 &(port->tx_conf[qi]));
2547 /* Fail to setup tx queue, return */
2548 if (rte_atomic16_cmpset(&(port->port_status),
2550 RTE_PORT_STOPPED) == 0)
2551 printf("Port %d can not be set back "
2552 "to stopped\n", pi);
2553 printf("Fail to configure port %d tx queues\n",
2555 /* try to reconfigure queues next time */
2556 port->need_reconfig_queues = 1;
2559 for (qi = 0; qi < nb_rxq; qi++) {
2560 /* setup rx queues */
2561 if ((numa_support) &&
2562 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2563 struct rte_mempool * mp =
2565 (rxring_numa[pi], 0);
2567 printf("Failed to setup RX queue:"
2568 "No mempool allocation"
2569 " on the socket %d\n",
2574 diag = rx_queue_setup(pi, qi,
2575 port->nb_rx_desc[qi],
2577 &(port->rx_conf[qi]),
2580 struct rte_mempool *mp =
2582 (port->socket_id, 0);
2584 printf("Failed to setup RX queue:"
2585 "No mempool allocation"
2586 " on the socket %d\n",
2590 diag = rx_queue_setup(pi, qi,
2591 port->nb_rx_desc[qi],
2593 &(port->rx_conf[qi]),
2599 /* Fail to setup rx queue, return */
2600 if (rte_atomic16_cmpset(&(port->port_status),
2602 RTE_PORT_STOPPED) == 0)
2603 printf("Port %d can not be set back "
2604 "to stopped\n", pi);
2605 printf("Fail to configure port %d rx queues\n",
2607 /* try to reconfigure queues next time */
2608 port->need_reconfig_queues = 1;
2611 /* setup hairpin queues */
2612 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2615 configure_rxtx_dump_callbacks(verbose_level);
2617 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2621 "Port %d: Failed to disable Ptype parsing\n",
2629 diag = rte_eth_dev_start(pi);
2631 printf("Fail to start port %d: %s\n", pi,
2632 rte_strerror(-diag));
2634 /* Fail to setup rx queue, return */
2635 if (rte_atomic16_cmpset(&(port->port_status),
2636 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2637 printf("Port %d can not be set back to "
2642 if (rte_atomic16_cmpset(&(port->port_status),
2643 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2644 printf("Port %d can not be set into started\n", pi);
2646 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2647 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2648 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2649 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2650 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2652 /* at least one port started, need checking link status */
2653 need_check_link_status = 1;
2658 if (need_check_link_status == 1 && !no_link_check)
2659 check_all_ports_link_status(RTE_PORT_ALL);
2660 else if (need_check_link_status == 0)
2661 printf("Please stop the ports first\n");
2663 if (hairpin_mode & 0xf) {
2667 /* bind all started hairpin ports */
2668 for (i = 0; i < cfg_pi; i++) {
2670 /* bind current Tx to all peer Rx */
2671 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2672 RTE_MAX_ETHPORTS, 1);
2675 for (j = 0; j < peer_pi; j++) {
2676 if (!port_is_started(peer_pl[j]))
2678 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2680 printf("Error during binding hairpin"
2681 " Tx port %u to %u: %s\n",
2683 rte_strerror(-diag));
2687 /* bind all peer Tx to current Rx */
2688 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2689 RTE_MAX_ETHPORTS, 0);
2692 for (j = 0; j < peer_pi; j++) {
2693 if (!port_is_started(peer_pl[j]))
2695 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2697 printf("Error during binding hairpin"
2698 " Tx port %u to %u: %s\n",
2700 rte_strerror(-diag));
2712 stop_port(portid_t pid)
2715 struct rte_port *port;
2716 int need_check_link_status = 0;
2717 portid_t peer_pl[RTE_MAX_ETHPORTS];
2725 if (port_id_is_invalid(pid, ENABLED_WARN))
2728 printf("Stopping ports...\n");
2730 RTE_ETH_FOREACH_DEV(pi) {
2731 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2734 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2735 printf("Please remove port %d from forwarding configuration.\n", pi);
2739 if (port_is_bonding_slave(pi)) {
2740 printf("Please remove port %d from bonded device.\n", pi);
2745 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2746 RTE_PORT_HANDLING) == 0)
2749 if (hairpin_mode & 0xf) {
2752 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2753 /* unbind all peer Tx from current Rx */
2754 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2755 RTE_MAX_ETHPORTS, 0);
2758 for (j = 0; j < peer_pi; j++) {
2759 if (!port_is_started(peer_pl[j]))
2761 rte_eth_hairpin_unbind(peer_pl[j], pi);
2765 if (port->flow_list)
2766 port_flow_flush(pi);
2768 if (rte_eth_dev_stop(pi) != 0)
2769 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2772 if (rte_atomic16_cmpset(&(port->port_status),
2773 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2774 printf("Port %d can not be set into stopped\n", pi);
2775 need_check_link_status = 1;
2777 if (need_check_link_status && !no_link_check)
2778 check_all_ports_link_status(RTE_PORT_ALL);
2784 remove_invalid_ports_in(portid_t *array, portid_t *total)
2787 portid_t new_total = 0;
2789 for (i = 0; i < *total; i++)
2790 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2791 array[new_total] = array[i];
2798 remove_invalid_ports(void)
2800 remove_invalid_ports_in(ports_ids, &nb_ports);
2801 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2802 nb_cfg_ports = nb_fwd_ports;
2806 close_port(portid_t pid)
2809 struct rte_port *port;
2811 if (port_id_is_invalid(pid, ENABLED_WARN))
2814 printf("Closing ports...\n");
2816 RTE_ETH_FOREACH_DEV(pi) {
2817 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2820 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2821 printf("Please remove port %d from forwarding configuration.\n", pi);
2825 if (port_is_bonding_slave(pi)) {
2826 printf("Please remove port %d from bonded device.\n", pi);
2831 if (rte_atomic16_cmpset(&(port->port_status),
2832 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2833 printf("Port %d is already closed\n", pi);
2837 port_flow_flush(pi);
2838 rte_eth_dev_close(pi);
2841 remove_invalid_ports();
2846 reset_port(portid_t pid)
2850 struct rte_port *port;
2852 if (port_id_is_invalid(pid, ENABLED_WARN))
2855 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2856 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2857 printf("Can not reset port(s), please stop port(s) first.\n");
2861 printf("Resetting ports...\n");
2863 RTE_ETH_FOREACH_DEV(pi) {
2864 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2867 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2868 printf("Please remove port %d from forwarding "
2869 "configuration.\n", pi);
2873 if (port_is_bonding_slave(pi)) {
2874 printf("Please remove port %d from bonded device.\n",
2879 diag = rte_eth_dev_reset(pi);
2882 port->need_reconfig = 1;
2883 port->need_reconfig_queues = 1;
2885 printf("Failed to reset port %d. diag=%d\n", pi, diag);
2893 attach_port(char *identifier)
2896 struct rte_dev_iterator iterator;
2898 printf("Attaching a new port...\n");
2900 if (identifier == NULL) {
2901 printf("Invalid parameters are specified\n");
2905 if (rte_dev_probe(identifier) < 0) {
2906 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2910 /* first attach mode: event */
2911 if (setup_on_probe_event) {
2912 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2913 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2914 if (ports[pi].port_status == RTE_PORT_HANDLING &&
2915 ports[pi].need_setup != 0)
2916 setup_attached_port(pi);
2920 /* second attach mode: iterator */
2921 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2922 /* setup ports matching the devargs used for probing */
2923 if (port_is_forwarding(pi))
2924 continue; /* port was already attached before */
2925 setup_attached_port(pi);
2930 setup_attached_port(portid_t pi)
2932 unsigned int socket_id;
2935 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2936 /* if socket_id is invalid, set to the first available socket. */
2937 if (check_socket_id(socket_id) < 0)
2938 socket_id = socket_ids[0];
2939 reconfig(pi, socket_id);
2940 ret = rte_eth_promiscuous_enable(pi);
2942 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2943 pi, rte_strerror(-ret));
2945 ports_ids[nb_ports++] = pi;
2946 fwd_ports_ids[nb_fwd_ports++] = pi;
2947 nb_cfg_ports = nb_fwd_ports;
2948 ports[pi].need_setup = 0;
2949 ports[pi].port_status = RTE_PORT_STOPPED;
2951 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2956 detach_device(struct rte_device *dev)
2961 printf("Device already removed\n");
2965 printf("Removing a device...\n");
2967 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2968 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2969 if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2970 printf("Port %u not stopped\n", sibling);
2973 port_flow_flush(sibling);
2977 if (rte_dev_remove(dev) < 0) {
2978 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2981 remove_invalid_ports();
2983 printf("Device is detached\n");
2984 printf("Now total ports is %d\n", nb_ports);
2990 detach_port_device(portid_t port_id)
2992 if (port_id_is_invalid(port_id, ENABLED_WARN))
2995 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2996 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2997 printf("Port not stopped\n");
3000 printf("Port was not closed\n");
3003 detach_device(rte_eth_devices[port_id].device);
3007 detach_devargs(char *identifier)
3009 struct rte_dev_iterator iterator;
3010 struct rte_devargs da;
3013 printf("Removing a device...\n");
3015 memset(&da, 0, sizeof(da));
3016 if (rte_devargs_parsef(&da, "%s", identifier)) {
3017 printf("cannot parse identifier\n");
3021 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3022 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3023 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3024 printf("Port %u not stopped\n", port_id);
3025 rte_eth_iterator_cleanup(&iterator);
3026 rte_devargs_reset(&da);
3029 port_flow_flush(port_id);
3033 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3034 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3035 da.name, da.bus->name);
3036 rte_devargs_reset(&da);
3040 remove_invalid_ports();
3042 printf("Device %s is detached\n", identifier);
3043 printf("Now total ports is %d\n", nb_ports);
3045 rte_devargs_reset(&da);
3056 stop_packet_forwarding();
3058 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3060 if (mp_alloc_type == MP_ALLOC_ANON)
3061 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3065 if (ports != NULL) {
3067 RTE_ETH_FOREACH_DEV(pt_id) {
3068 printf("\nStopping port %d...\n", pt_id);
3072 RTE_ETH_FOREACH_DEV(pt_id) {
3073 printf("\nShutting down port %d...\n", pt_id);
3080 ret = rte_dev_event_monitor_stop();
3083 "fail to stop device event monitor.");
3087 ret = rte_dev_event_callback_unregister(NULL,
3088 dev_event_callback, NULL);
3091 "fail to unregister device event callback.\n");
3095 ret = rte_dev_hotplug_handle_disable();
3098 "fail to disable hotplug handling.\n");
3102 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3104 rte_mempool_free(mempools[i]);
3107 printf("\nBye...\n");
3110 typedef void (*cmd_func_t)(void);
3111 struct pmd_test_command {
3112 const char *cmd_name;
3113 cmd_func_t cmd_func;
3116 /* Check the link status of all ports in up to 9s, and print them finally */
3118 check_all_ports_link_status(uint32_t port_mask)
3120 #define CHECK_INTERVAL 100 /* 100ms */
3121 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3123 uint8_t count, all_ports_up, print_flag = 0;
3124 struct rte_eth_link link;
3126 char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3128 printf("Checking link statuses...\n");
3130 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3132 RTE_ETH_FOREACH_DEV(portid) {
3133 if ((port_mask & (1 << portid)) == 0)
3135 memset(&link, 0, sizeof(link));
3136 ret = rte_eth_link_get_nowait(portid, &link);
3139 if (print_flag == 1)
3140 printf("Port %u link get failed: %s\n",
3141 portid, rte_strerror(-ret));
3144 /* print link status if flag set */
3145 if (print_flag == 1) {
3146 rte_eth_link_to_str(link_status,
3147 sizeof(link_status), &link);
3148 printf("Port %d %s\n", portid, link_status);
3151 /* clear all_ports_up flag if any link down */
3152 if (link.link_status == ETH_LINK_DOWN) {
3157 /* after finally printing all link status, get out */
3158 if (print_flag == 1)
3161 if (all_ports_up == 0) {
3163 rte_delay_ms(CHECK_INTERVAL);
3166 /* set the print_flag if all ports up or timeout */
3167 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3177 rmv_port_callback(void *arg)
3179 int need_to_start = 0;
3180 int org_no_link_check = no_link_check;
3181 portid_t port_id = (intptr_t)arg;
3182 struct rte_device *dev;
3184 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3186 if (!test_done && port_is_forwarding(port_id)) {
3188 stop_packet_forwarding();
3192 no_link_check = org_no_link_check;
3194 /* Save rte_device pointer before closing ethdev port */
3195 dev = rte_eth_devices[port_id].device;
3196 close_port(port_id);
3197 detach_device(dev); /* might be already removed or have more ports */
3200 start_packet_forwarding(0);
3203 /* This function is used by the interrupt thread */
3205 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3208 RTE_SET_USED(param);
3209 RTE_SET_USED(ret_param);
3211 if (type >= RTE_ETH_EVENT_MAX) {
3212 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3213 port_id, __func__, type);
3215 } else if (event_print_mask & (UINT32_C(1) << type)) {
3216 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3217 eth_event_desc[type]);
3222 case RTE_ETH_EVENT_NEW:
3223 ports[port_id].need_setup = 1;
3224 ports[port_id].port_status = RTE_PORT_HANDLING;
3226 case RTE_ETH_EVENT_INTR_RMV:
3227 if (port_id_is_invalid(port_id, DISABLED_WARN))
3229 if (rte_eal_alarm_set(100000,
3230 rmv_port_callback, (void *)(intptr_t)port_id))
3231 fprintf(stderr, "Could not set up deferred device removal\n");
3233 case RTE_ETH_EVENT_DESTROY:
3234 ports[port_id].port_status = RTE_PORT_CLOSED;
3235 printf("Port %u is closed\n", port_id);
3244 register_eth_event_callback(void)
3247 enum rte_eth_event_type event;
3249 for (event = RTE_ETH_EVENT_UNKNOWN;
3250 event < RTE_ETH_EVENT_MAX; event++) {
3251 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3256 TESTPMD_LOG(ERR, "Failed to register callback for "
3257 "%s event\n", eth_event_desc[event]);
3265 /* This function is used by the interrupt thread */
3267 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3268 __rte_unused void *arg)
3273 if (type >= RTE_DEV_EVENT_MAX) {
3274 fprintf(stderr, "%s called upon invalid event %d\n",
3280 case RTE_DEV_EVENT_REMOVE:
3281 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3283 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3285 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3290 * Because the user's callback is invoked in eal interrupt
3291 * callback, the interrupt callback need to be finished before
3292 * it can be unregistered when detaching device. So finish
3293 * callback soon and use a deferred removal to detach device
3294 * is need. It is a workaround, once the device detaching be
3295 * moved into the eal in the future, the deferred removal could
3298 if (rte_eal_alarm_set(100000,
3299 rmv_port_callback, (void *)(intptr_t)port_id))
3301 "Could not set up deferred device removal\n");
3303 case RTE_DEV_EVENT_ADD:
3304 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3306 /* TODO: After finish kernel driver binding,
3307 * begin to attach port.
3316 rxtx_port_config(struct rte_port *port)
3321 for (qid = 0; qid < nb_rxq; qid++) {
3322 offloads = port->rx_conf[qid].offloads;
3323 port->rx_conf[qid] = port->dev_info.default_rxconf;
3325 port->rx_conf[qid].offloads = offloads;
3327 /* Check if any Rx parameters have been passed */
3328 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3329 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3331 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3332 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3334 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3335 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3337 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3338 port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3340 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3341 port->rx_conf[qid].rx_drop_en = rx_drop_en;
3343 port->nb_rx_desc[qid] = nb_rxd;
3346 for (qid = 0; qid < nb_txq; qid++) {
3347 offloads = port->tx_conf[qid].offloads;
3348 port->tx_conf[qid] = port->dev_info.default_txconf;
3350 port->tx_conf[qid].offloads = offloads;
3352 /* Check if any Tx parameters have been passed */
3353 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3354 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3356 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3357 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3359 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3360 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3362 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3363 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3365 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3366 port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3368 port->nb_tx_desc[qid] = nb_txd;
3373 * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3374 * MTU is also aligned if JUMBO_FRAME offload is not set.
3376 * port->dev_info should be set before calling this function.
3378 * return 0 on success, negative on error
3381 update_jumbo_frame_offload(portid_t portid)
3383 struct rte_port *port = &ports[portid];
3384 uint32_t eth_overhead;
3385 uint64_t rx_offloads;
3389 /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3390 if (port->dev_info.max_mtu != UINT16_MAX &&
3391 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3392 eth_overhead = port->dev_info.max_rx_pktlen -
3393 port->dev_info.max_mtu;
3395 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3397 rx_offloads = port->dev_conf.rxmode.offloads;
3399 /* Default config value is 0 to use PMD specific overhead */
3400 if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3401 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3403 if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3404 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3407 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3408 printf("Frame size (%u) is not supported by port %u\n",
3409 port->dev_conf.rxmode.max_rx_pkt_len,
3413 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3417 if (rx_offloads != port->dev_conf.rxmode.offloads) {
3420 port->dev_conf.rxmode.offloads = rx_offloads;
3422 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3423 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3425 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3427 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3431 /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3432 * if unset do it here
3434 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3435 ret = rte_eth_dev_set_mtu(portid,
3436 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3438 printf("Failed to set MTU to %u for port %u\n",
3439 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3447 init_port_config(void)
3450 struct rte_port *port;
3453 RTE_ETH_FOREACH_DEV(pid) {
3455 port->dev_conf.fdir_conf = fdir_conf;
3457 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3462 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3463 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3464 rss_hf & port->dev_info.flow_type_rss_offloads;
3466 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3467 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3470 if (port->dcb_flag == 0) {
3471 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3472 port->dev_conf.rxmode.mq_mode =
3473 (enum rte_eth_rx_mq_mode)
3474 (rx_mq_mode & ETH_MQ_RX_RSS);
3476 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3479 rxtx_port_config(port);
3481 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3485 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3486 rte_pmd_ixgbe_bypass_init(pid);
3489 if (lsc_interrupt &&
3490 (rte_eth_devices[pid].data->dev_flags &
3491 RTE_ETH_DEV_INTR_LSC))
3492 port->dev_conf.intr_conf.lsc = 1;
3493 if (rmv_interrupt &&
3494 (rte_eth_devices[pid].data->dev_flags &
3495 RTE_ETH_DEV_INTR_RMV))
3496 port->dev_conf.intr_conf.rmv = 1;
3500 void set_port_slave_flag(portid_t slave_pid)
3502 struct rte_port *port;
3504 port = &ports[slave_pid];
3505 port->slave_flag = 1;
3508 void clear_port_slave_flag(portid_t slave_pid)
3510 struct rte_port *port;
3512 port = &ports[slave_pid];
3513 port->slave_flag = 0;
3516 uint8_t port_is_bonding_slave(portid_t slave_pid)
3518 struct rte_port *port;
3520 port = &ports[slave_pid];
3521 if ((rte_eth_devices[slave_pid].data->dev_flags &
3522 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3527 const uint16_t vlan_tags[] = {
3528 0, 1, 2, 3, 4, 5, 6, 7,
3529 8, 9, 10, 11, 12, 13, 14, 15,
3530 16, 17, 18, 19, 20, 21, 22, 23,
3531 24, 25, 26, 27, 28, 29, 30, 31
3535 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3536 enum dcb_mode_enable dcb_mode,
3537 enum rte_eth_nb_tcs num_tcs,
3542 struct rte_eth_rss_conf rss_conf;
3545 * Builds up the correct configuration for dcb+vt based on the vlan tags array
3546 * given above, and the number of traffic classes available for use.
3548 if (dcb_mode == DCB_VT_ENABLED) {
3549 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3550 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3551 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3552 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3554 /* VMDQ+DCB RX and TX configurations */
3555 vmdq_rx_conf->enable_default_pool = 0;
3556 vmdq_rx_conf->default_pool = 0;
3557 vmdq_rx_conf->nb_queue_pools =
3558 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3559 vmdq_tx_conf->nb_queue_pools =
3560 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3562 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3563 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3564 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3565 vmdq_rx_conf->pool_map[i].pools =
3566 1 << (i % vmdq_rx_conf->nb_queue_pools);
3568 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3569 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3570 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3573 /* set DCB mode of RX and TX of multiple queues */
3574 eth_conf->rxmode.mq_mode =
3575 (enum rte_eth_rx_mq_mode)
3576 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3577 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3579 struct rte_eth_dcb_rx_conf *rx_conf =
3580 ð_conf->rx_adv_conf.dcb_rx_conf;
3581 struct rte_eth_dcb_tx_conf *tx_conf =
3582 ð_conf->tx_adv_conf.dcb_tx_conf;
3584 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3586 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3590 rx_conf->nb_tcs = num_tcs;
3591 tx_conf->nb_tcs = num_tcs;
3593 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3594 rx_conf->dcb_tc[i] = i % num_tcs;
3595 tx_conf->dcb_tc[i] = i % num_tcs;
3598 eth_conf->rxmode.mq_mode =
3599 (enum rte_eth_rx_mq_mode)
3600 (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3601 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3602 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3606 eth_conf->dcb_capability_en =
3607 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3609 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3615 init_port_dcb_config(portid_t pid,
3616 enum dcb_mode_enable dcb_mode,
3617 enum rte_eth_nb_tcs num_tcs,
3620 struct rte_eth_conf port_conf;
3621 struct rte_port *rte_port;
3625 rte_port = &ports[pid];
3627 memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3628 /* Enter DCB configuration status */
3631 port_conf.rxmode = rte_port->dev_conf.rxmode;
3632 port_conf.txmode = rte_port->dev_conf.txmode;
3634 /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3635 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3638 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3640 /* re-configure the device . */
3641 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3645 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3649 /* If dev_info.vmdq_pool_base is greater than 0,
3650 * the queue id of vmdq pools is started after pf queues.
3652 if (dcb_mode == DCB_VT_ENABLED &&
3653 rte_port->dev_info.vmdq_pool_base > 0) {
3654 printf("VMDQ_DCB multi-queue mode is nonsensical"
3655 " for port %d.", pid);
3659 /* Assume the ports in testpmd have the same dcb capability
3660 * and has the same number of rxq and txq in dcb mode
3662 if (dcb_mode == DCB_VT_ENABLED) {
3663 if (rte_port->dev_info.max_vfs > 0) {
3664 nb_rxq = rte_port->dev_info.nb_rx_queues;
3665 nb_txq = rte_port->dev_info.nb_tx_queues;
3667 nb_rxq = rte_port->dev_info.max_rx_queues;
3668 nb_txq = rte_port->dev_info.max_tx_queues;
3671 /*if vt is disabled, use all pf queues */
3672 if (rte_port->dev_info.vmdq_pool_base == 0) {
3673 nb_rxq = rte_port->dev_info.max_rx_queues;
3674 nb_txq = rte_port->dev_info.max_tx_queues;
3676 nb_rxq = (queueid_t)num_tcs;
3677 nb_txq = (queueid_t)num_tcs;
3681 rx_free_thresh = 64;
3683 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3685 rxtx_port_config(rte_port);
3687 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3688 for (i = 0; i < RTE_DIM(vlan_tags); i++)
3689 rx_vft_set(pid, vlan_tags[i], 1);
3691 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3695 rte_port->dcb_flag = 1;
3705 /* Configuration of Ethernet ports. */
3706 ports = rte_zmalloc("testpmd: ports",
3707 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3708 RTE_CACHE_LINE_SIZE);
3709 if (ports == NULL) {
3710 rte_exit(EXIT_FAILURE,
3711 "rte_zmalloc(%d struct rte_port) failed\n",
3714 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3715 LIST_INIT(&ports[i].flow_tunnel_list);
3716 /* Initialize ports NUMA structures */
3717 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3718 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3719 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3733 const char clr[] = { 27, '[', '2', 'J', '\0' };
3734 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3736 /* Clear screen and move to top left */
3737 printf("%s%s", clr, top_left);
3739 printf("\nPort statistics ====================================");
3740 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3741 nic_stats_display(fwd_ports_ids[i]);
3747 signal_handler(int signum)
3749 if (signum == SIGINT || signum == SIGTERM) {
3750 printf("\nSignal %d received, preparing to exit...\n",
3752 #ifdef RTE_LIB_PDUMP
3753 /* uninitialize packet capture framework */
3756 #ifdef RTE_LIB_LATENCYSTATS
3757 if (latencystats_enabled != 0)
3758 rte_latencystats_uninit();
3761 /* Set flag to indicate the force termination. */
3763 /* exit with the expected status */
3764 signal(signum, SIG_DFL);
3765 kill(getpid(), signum);
3770 main(int argc, char** argv)
3777 signal(SIGINT, signal_handler);
3778 signal(SIGTERM, signal_handler);
3780 testpmd_logtype = rte_log_register("testpmd");
3781 if (testpmd_logtype < 0)
3782 rte_exit(EXIT_FAILURE, "Cannot register log type");
3783 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3785 diag = rte_eal_init(argc, argv);
3787 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3788 rte_strerror(rte_errno));
3790 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3791 rte_exit(EXIT_FAILURE,
3792 "Secondary process type not supported.\n");
3794 ret = register_eth_event_callback();
3796 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3798 #ifdef RTE_LIB_PDUMP
3799 /* initialize packet capture framework */
3804 RTE_ETH_FOREACH_DEV(port_id) {
3805 ports_ids[count] = port_id;
3808 nb_ports = (portid_t) count;
3810 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3812 /* allocate port structures, and init them */
3815 set_def_fwd_config();
3817 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3818 "Check the core mask argument\n");
3820 /* Bitrate/latency stats disabled by default */
3821 #ifdef RTE_LIB_BITRATESTATS
3822 bitrate_enabled = 0;
3824 #ifdef RTE_LIB_LATENCYSTATS
3825 latencystats_enabled = 0;
3828 /* on FreeBSD, mlockall() is disabled by default */
3829 #ifdef RTE_EXEC_ENV_FREEBSD
3838 launch_args_parse(argc, argv);
3840 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3841 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3845 if (tx_first && interactive)
3846 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3847 "interactive mode.\n");
3849 if (tx_first && lsc_interrupt) {
3850 printf("Warning: lsc_interrupt needs to be off when "
3851 " using tx_first. Disabling.\n");
3855 if (!nb_rxq && !nb_txq)
3856 printf("Warning: Either rx or tx queues should be non-zero\n");
3858 if (nb_rxq > 1 && nb_rxq > nb_txq)
3859 printf("Warning: nb_rxq=%d enables RSS configuration, "
3860 "but nb_txq=%d will prevent to fully test it.\n",
3866 ret = rte_dev_hotplug_handle_enable();
3869 "fail to enable hotplug handling.");
3873 ret = rte_dev_event_monitor_start();
3876 "fail to start device event monitoring.");
3880 ret = rte_dev_event_callback_register(NULL,
3881 dev_event_callback, NULL);
3884 "fail to register device event callback\n");
3889 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3890 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3892 /* set all ports to promiscuous mode by default */
3893 RTE_ETH_FOREACH_DEV(port_id) {
3894 ret = rte_eth_promiscuous_enable(port_id);
3896 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3897 port_id, rte_strerror(-ret));
3900 /* Init metrics library */
3901 rte_metrics_init(rte_socket_id());
3903 #ifdef RTE_LIB_LATENCYSTATS
3904 if (latencystats_enabled != 0) {
3905 int ret = rte_latencystats_init(1, NULL);
3907 printf("Warning: latencystats init()"
3908 " returned error %d\n", ret);
3909 printf("Latencystats running on lcore %d\n",
3910 latencystats_lcore_id);
3914 /* Setup bitrate stats */
3915 #ifdef RTE_LIB_BITRATESTATS
3916 if (bitrate_enabled != 0) {
3917 bitrate_data = rte_stats_bitrate_create();
3918 if (bitrate_data == NULL)
3919 rte_exit(EXIT_FAILURE,
3920 "Could not allocate bitrate data.\n");
3921 rte_stats_bitrate_reg(bitrate_data);
3925 #ifdef RTE_LIB_CMDLINE
3926 if (strlen(cmdline_filename) != 0)
3927 cmdline_read_from_file(cmdline_filename);
3929 if (interactive == 1) {
3931 printf("Start automatic packet forwarding\n");
3932 start_packet_forwarding(0);
3944 printf("No commandline core given, start packet forwarding\n");
3945 start_packet_forwarding(tx_first);
3946 if (stats_period != 0) {
3947 uint64_t prev_time = 0, cur_time, diff_time = 0;
3948 uint64_t timer_period;
3950 /* Convert to number of cycles */
3951 timer_period = stats_period * rte_get_timer_hz();
3953 while (f_quit == 0) {
3954 cur_time = rte_get_timer_cycles();
3955 diff_time += cur_time - prev_time;
3957 if (diff_time >= timer_period) {
3959 /* Reset the timer */
3962 /* Sleep to avoid unnecessary checks */
3963 prev_time = cur_time;
3968 printf("Press enter to exit\n");
3969 rc = read(0, &c, 1);
3975 ret = rte_eal_cleanup();
3977 rte_exit(EXIT_FAILURE,
3978 "EAL cleanup failed: %s\n", strerror(-ret));
3980 return EXIT_SUCCESS;