app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_NET_IXGBE
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIB_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #ifdef RTE_LIB_METRICS
  58 #include <rte_metrics.h>
  59 #endif
  60 #ifdef RTE_LIB_BITRATESTATS
  61 #include <rte_bitrate.h>
  62 #endif
  63 #ifdef RTE_LIB_LATENCYSTATS
  64 #include <rte_latencystats.h>
  65 #endif
  66 #ifdef RTE_EXEC_ENV_WINDOWS
  67 #include <process.h>
  68 #endif
  69 #ifdef RTE_NET_BOND
  70 #include <rte_eth_bond.h>
  71 #endif
  72
  73 #include "testpmd.h"
  74
  75 #ifndef MAP_HUGETLB
  76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  77 #define HUGE_FLAG (0x40000)
  78 #else
  79 #define HUGE_FLAG MAP_HUGETLB
  80 #endif
  81
  82 #ifndef MAP_HUGE_SHIFT
  83 /* older kernels (or FreeBSD) will not have this define */
  84 #define HUGE_SHIFT (26)
  85 #else
  86 #define HUGE_SHIFT MAP_HUGE_SHIFT
  87 #endif
  88
  89 #define EXTMEM_HEAP_NAME "extmem"
  90 /*
  91  * Zone size with the malloc overhead (max of debug and release variants)
  92  * must fit into the smallest supported hugepage size (2M),
  93  * so that an IOVA-contiguous zone of this size can always be allocated
  94  * if there are free 2M hugepages.
  95  */
  96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE)
  97
  98 uint16_t verbose_level = 0; /**< Silent by default. */
  99 int testpmd_logtype; /**< Log type for testpmd logs */
 100
 101 /* use main core for command line ? */
 102 uint8_t interactive = 0;
 103 uint8_t auto_start = 0;
 104 uint8_t tx_first;
 105 char cmdline_filename[PATH_MAX] = {0};
 106
 107 /*
 108  * NUMA support configuration.
 109  * When set, the NUMA support attempts to dispatch the allocation of the
 110  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 111  * probed ports among the CPU sockets 0 and 1.
 112  * Otherwise, all memory is allocated from CPU socket 0.
 113  */
 114 uint8_t numa_support = 1; /**< numa enabled by default */
 115
 116 /*
 117  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 118  * not configured.
 119  */
 120 uint8_t socket_num = UMA_NO_CONFIG;
 121
 122 /*
 123  * Select mempool allocation type:
 124  * - native: use regular DPDK memory
 125  * - anon: use regular DPDK memory to create mempool, but populate using
 126  *         anonymous memory (may not be IOVA-contiguous)
 127  * - xmem: use externally allocated hugepage memory
 128  */
 129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 130
 131 /*
 132  * Store specified sockets on which memory pool to be used by ports
 133  * is allocated.
 134  */
 135 uint8_t port_numa[RTE_MAX_ETHPORTS];
 136
 137 /*
 138  * Store specified sockets on which RX ring to be used by ports
 139  * is allocated.
 140  */
 141 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 142
 143 /*
 144  * Store specified sockets on which TX ring to be used by ports
 145  * is allocated.
 146  */
 147 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 148
 149 /*
 150  * Record the Ethernet address of peer target ports to which packets are
 151  * forwarded.
 152  * Must be instantiated with the ethernet addresses of peer traffic generator
 153  * ports.
 154  */
 155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 156 portid_t nb_peer_eth_addrs = 0;
 157
 158 /*
 159  * Probed Target Environment.
 160  */
 161 struct rte_port *ports;        /**< For all probed ethernet ports. */
 162 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 164 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 165
 166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 167
 168 /*
 169  * Test Forwarding Configuration.
 170  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 171  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 172  */
 173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 175 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 176 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 177
 178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 180
 181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 182 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 183
 184 /*
 185  * Forwarding engines.
 186  */
 187 struct fwd_engine * fwd_engines[] = {
 188         &io_fwd_engine,
 189         &mac_fwd_engine,
 190         &mac_swap_engine,
 191         &flow_gen_engine,
 192         &rx_only_engine,
 193         &tx_only_engine,
 194         &csum_fwd_engine,
 195         &icmp_echo_engine,
 196         &noisy_vnf_engine,
 197         &five_tuple_swap_fwd_engine,
 198 #ifdef RTE_LIBRTE_IEEE1588
 199         &ieee1588_fwd_engine,
 200 #endif
 201         &shared_rxq_engine,
 202         NULL,
 203 };
 204
 205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 206 uint16_t mempool_flags;
 207
 208 struct fwd_config cur_fwd_config;
 209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 210 uint32_t retry_enabled;
 211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 213
 214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 216         DEFAULT_MBUF_DATA_SIZE
 217 }; /**< Mbuf data space size. */
 218 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 219                                       * specified on command-line. */
 220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 221
 222 /** Extended statistics to show. */
 223 struct rte_eth_xstat_name *xstats_display;
 224
 225 unsigned int xstats_display_num; /**< Size of extended statistics to show */
 226
 227 /*
 228  * In container, it cannot terminate the process which running with 'stats-period'
 229  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 230  */
 231 uint8_t f_quit;
 232
 233 /*
 234  * Max Rx frame size, set by '--max-pkt-len' parameter.
 235  */
 236 uint32_t max_rx_pkt_len;
 237
 238 /*
 239  * Configuration of packet segments used to scatter received packets
 240  * if some of split features is configured.
 241  */
 242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 243 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 245 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 246
 247 /*
 248  * Configuration of packet segments used by the "txonly" processing engine.
 249  */
 250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 252         TXONLY_DEF_PACKET_LEN,
 253 };
 254 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 255
 256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 257 /**< Split policy for packets to TX. */
 258
 259 uint8_t txonly_multi_flow;
 260 /**< Whether multiple flows are generated in TXONLY mode. */
 261
 262 uint32_t tx_pkt_times_inter;
 263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 264
 265 uint32_t tx_pkt_times_intra;
 266 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 267
 268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
 271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 272
 273 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 274 uint8_t dcb_config = 0;
 275
 276 /*
 277  * Configurable number of RX/TX queues.
 278  */
 279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 282
 283 /*
 284  * Configurable number of RX/TX ring descriptors.
 285  * Defaults are supplied by drivers via ethdev.
 286  */
 287 #define RTE_TEST_RX_DESC_DEFAULT 0
 288 #define RTE_TEST_TX_DESC_DEFAULT 0
 289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 291
 292 #define RTE_PMD_PARAM_UNSET -1
 293 /*
 294  * Configurable values of RX and TX ring threshold registers.
 295  */
 296
 297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 300
 301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 304
 305 /*
 306  * Configurable value of RX free threshold.
 307  */
 308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 309
 310 /*
 311  * Configurable value of RX drop enable.
 312  */
 313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 314
 315 /*
 316  * Configurable value of TX free threshold.
 317  */
 318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 319
 320 /*
 321  * Configurable value of TX RS bit threshold.
 322  */
 323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 324
 325 /*
 326  * Configurable value of buffered packets before sending.
 327  */
 328 uint16_t noisy_tx_sw_bufsz;
 329
 330 /*
 331  * Configurable value of packet buffer timeout.
 332  */
 333 uint16_t noisy_tx_sw_buf_flush_time;
 334
 335 /*
 336  * Configurable value for size of VNF internal memory area
 337  * used for simulating noisy neighbour behaviour
 338  */
 339 uint64_t noisy_lkup_mem_sz;
 340
 341 /*
 342  * Configurable value of number of random writes done in
 343  * VNF simulation memory area.
 344  */
 345 uint64_t noisy_lkup_num_writes;
 346
 347 /*
 348  * Configurable value of number of random reads done in
 349  * VNF simulation memory area.
 350  */
 351 uint64_t noisy_lkup_num_reads;
 352
 353 /*
 354  * Configurable value of number of random reads/writes done in
 355  * VNF simulation memory area.
 356  */
 357 uint64_t noisy_lkup_num_reads_writes;
 358
 359 /*
 360  * Receive Side Scaling (RSS) configuration.
 361  */
 362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */
 363
 364 /*
 365  * Port topology configuration
 366  */
 367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 368
 369 /*
 370  * Avoids to flush all the RX streams before starts forwarding.
 371  */
 372 uint8_t no_flush_rx = 0; /* flush by default */
 373
 374 /*
 375  * Flow API isolated mode.
 376  */
 377 uint8_t flow_isolate_all;
 378
 379 /*
 380  * Avoids to check link status when starting/stopping a port.
 381  */
 382 uint8_t no_link_check = 0; /* check by default */
 383
 384 /*
 385  * Don't automatically start all ports in interactive mode.
 386  */
 387 uint8_t no_device_start = 0;
 388
 389 /*
 390  * Enable link status change notification
 391  */
 392 uint8_t lsc_interrupt = 1; /* enabled by default */
 393
 394 /*
 395  * Enable device removal notification.
 396  */
 397 uint8_t rmv_interrupt = 1; /* enabled by default */
 398
 399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 400
 401 /* After attach, port setup is called on event or by iterator */
 402 bool setup_on_probe_event = true;
 403
 404 /* Clear ptypes on port initialization. */
 405 uint8_t clear_ptypes = true;
 406
 407 /* Hairpin ports configuration mode. */
 408 uint16_t hairpin_mode;
 409
 410 /* Pretty printing of ethdev events */
 411 static const char * const eth_event_desc[] = {
 412         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 413         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 414         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 415         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 416         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 417         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 418         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 419         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 420         [RTE_ETH_EVENT_NEW] = "device probed",
 421         [RTE_ETH_EVENT_DESTROY] = "device released",
 422         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 423         [RTE_ETH_EVENT_MAX] = NULL,
 424 };
 425
 426 /*
 427  * Display or mask ether events
 428  * Default to all events except VF_MBOX
 429  */
 430 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 431                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 432                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 433                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 434                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 435                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 436                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 437                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 438 /*
 439  * Decide if all memory are locked for performance.
 440  */
 441 int do_mlockall = 0;
 442
 443 /*
 444  * NIC bypass mode configuration options.
 445  */
 446
 447 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 448 /* The NIC bypass watchdog timeout. */
 449 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 450 #endif
 451
 452
 453 #ifdef RTE_LIB_LATENCYSTATS
 454
 455 /*
 456  * Set when latency stats is enabled in the commandline
 457  */
 458 uint8_t latencystats_enabled;
 459
 460 /*
 461  * Lcore ID to service latency statistics.
 462  */
 463 lcoreid_t latencystats_lcore_id = -1;
 464
 465 #endif
 466
 467 /*
 468  * Ethernet device configuration.
 469  */
 470 struct rte_eth_rxmode rx_mode;
 471
 472 struct rte_eth_txmode tx_mode = {
 473         .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
 474 };
 475
 476 struct rte_eth_fdir_conf fdir_conf = {
 477         .mode = RTE_FDIR_MODE_NONE,
 478         .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
 479         .status = RTE_FDIR_REPORT_STATUS,
 480         .mask = {
 481                 .vlan_tci_mask = 0xFFEF,
 482                 .ipv4_mask     = {
 483                         .src_ip = 0xFFFFFFFF,
 484                         .dst_ip = 0xFFFFFFFF,
 485                 },
 486                 .ipv6_mask     = {
 487                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 488                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 489                 },
 490                 .src_port_mask = 0xFFFF,
 491                 .dst_port_mask = 0xFFFF,
 492                 .mac_addr_byte_mask = 0xFF,
 493                 .tunnel_type_mask = 1,
 494                 .tunnel_id_mask = 0xFFFFFFFF,
 495         },
 496         .drop_queue = 127,
 497 };
 498
 499 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 500
 501 /*
 502  * Display zero values by default for xstats
 503  */
 504 uint8_t xstats_hide_zero;
 505
 506 /*
 507  * Measure of CPU cycles disabled by default
 508  */
 509 uint8_t record_core_cycles;
 510
 511 /*
 512  * Display of RX and TX bursts disabled by default
 513  */
 514 uint8_t record_burst_stats;
 515
 516 /*
 517  * Number of ports per shared Rx queue group, 0 disable.
 518  */
 519 uint32_t rxq_share;
 520
 521 unsigned int num_sockets = 0;
 522 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 523
 524 #ifdef RTE_LIB_BITRATESTATS
 525 /* Bitrate statistics */
 526 struct rte_stats_bitrates *bitrate_data;
 527 lcoreid_t bitrate_lcore_id;
 528 uint8_t bitrate_enabled;
 529 #endif
 530
 531 #ifdef RTE_LIB_GRO
 532 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 533 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 534 #endif
 535
 536 /*
 537  * hexadecimal bitmask of RX mq mode can be enabled.
 538  */
 539 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
 540
 541 /*
 542  * Used to set forced link speed
 543  */
 544 uint32_t eth_link_speed;
 545
 546 /*
 547  * ID of the current process in multi-process, used to
 548  * configure the queues to be polled.
 549  */
 550 int proc_id;
 551
 552 /*
 553  * Number of processes in multi-process, used to
 554  * configure the queues to be polled.
 555  */
 556 unsigned int num_procs = 1;
 557
 558 static void
 559 eth_rx_metadata_negotiate_mp(uint16_t port_id)
 560 {
 561         uint64_t rx_meta_features = 0;
 562         int ret;
 563
 564         if (!is_proc_primary())
 565                 return;
 566
 567         rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
 568         rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
 569         rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
 570
 571         ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
 572         if (ret == 0) {
 573                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
 574                         TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
 575                                     port_id);
 576                 }
 577
 578                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
 579                         TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
 580                                     port_id);
 581                 }
 582
 583                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
 584                         TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
 585                                     port_id);
 586                 }
 587         } else if (ret != -ENOTSUP) {
 588                 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
 589                          port_id, rte_strerror(-ret));
 590         }
 591 }
 592
 593 static int
 594 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 595                       const struct rte_eth_conf *dev_conf)
 596 {
 597         if (is_proc_primary())
 598                 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
 599                                         dev_conf);
 600         return 0;
 601 }
 602
 603 static int
 604 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop)
 605 {
 606 #ifdef RTE_NET_BOND
 607
 608         portid_t slave_pids[RTE_MAX_ETHPORTS];
 609         struct rte_port *port;
 610         int num_slaves;
 611         portid_t slave_pid;
 612         int i;
 613
 614         num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids,
 615                                                 RTE_MAX_ETHPORTS);
 616         if (num_slaves < 0) {
 617                 fprintf(stderr, "Failed to get slave list for port = %u\n",
 618                         bond_pid);
 619                 return num_slaves;
 620         }
 621
 622         for (i = 0; i < num_slaves; i++) {
 623                 slave_pid = slave_pids[i];
 624                 port = &ports[slave_pid];
 625                 port->port_status =
 626                         is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED;
 627         }
 628 #else
 629         RTE_SET_USED(bond_pid);
 630         RTE_SET_USED(is_stop);
 631 #endif
 632         return 0;
 633 }
 634
 635 static int
 636 eth_dev_start_mp(uint16_t port_id)
 637 {
 638         int ret;
 639
 640         if (is_proc_primary()) {
 641                 ret = rte_eth_dev_start(port_id);
 642                 if (ret != 0)
 643                         return ret;
 644
 645                 struct rte_port *port = &ports[port_id];
 646
 647                 /*
 648                  * Starting a bonded port also starts all slaves under the bonded
 649                  * device. So if this port is bond device, we need to modify the
 650                  * port status of these slaves.
 651                  */
 652                 if (port->bond_flag == 1)
 653                         return change_bonding_slave_port_status(port_id, false);
 654         }
 655
 656         return 0;
 657 }
 658
 659 static int
 660 eth_dev_stop_mp(uint16_t port_id)
 661 {
 662         int ret;
 663
 664         if (is_proc_primary()) {
 665                 ret = rte_eth_dev_stop(port_id);
 666                 if (ret != 0)
 667                         return ret;
 668
 669                 struct rte_port *port = &ports[port_id];
 670
 671                 /*
 672                  * Stopping a bonded port also stops all slaves under the bonded
 673                  * device. So if this port is bond device, we need to modify the
 674                  * port status of these slaves.
 675                  */
 676                 if (port->bond_flag == 1)
 677                         return change_bonding_slave_port_status(port_id, true);
 678         }
 679
 680         return 0;
 681 }
 682
 683 static void
 684 mempool_free_mp(struct rte_mempool *mp)
 685 {
 686         if (is_proc_primary())
 687                 rte_mempool_free(mp);
 688 }
 689
 690 static int
 691 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
 692 {
 693         if (is_proc_primary())
 694                 return rte_eth_dev_set_mtu(port_id, mtu);
 695
 696         return 0;
 697 }
 698
 699 /* Forward function declarations */
 700 static void setup_attached_port(portid_t pi);
 701 static void check_all_ports_link_status(uint32_t port_mask);
 702 static int eth_event_callback(portid_t port_id,
 703                               enum rte_eth_event_type type,
 704                               void *param, void *ret_param);
 705 static void dev_event_callback(const char *device_name,
 706                                 enum rte_dev_event_type type,
 707                                 void *param);
 708 static void fill_xstats_display_info(void);
 709
 710 /*
 711  * Check if all the ports are started.
 712  * If yes, return positive value. If not, return zero.
 713  */
 714 static int all_ports_started(void);
 715
 716 #ifdef RTE_LIB_GSO
 717 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 718 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 719 #endif
 720
 721 /* Holds the registered mbuf dynamic flags names. */
 722 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 723
 724
 725 /*
 726  * Helper function to check if socket is already discovered.
 727  * If yes, return positive value. If not, return zero.
 728  */
 729 int
 730 new_socket_id(unsigned int socket_id)
 731 {
 732         unsigned int i;
 733
 734         for (i = 0; i < num_sockets; i++) {
 735                 if (socket_ids[i] == socket_id)
 736                         return 0;
 737         }
 738         return 1;
 739 }
 740
 741 /*
 742  * Setup default configuration.
 743  */
 744 static void
 745 set_default_fwd_lcores_config(void)
 746 {
 747         unsigned int i;
 748         unsigned int nb_lc;
 749         unsigned int sock_num;
 750
 751         nb_lc = 0;
 752         for (i = 0; i < RTE_MAX_LCORE; i++) {
 753                 if (!rte_lcore_is_enabled(i))
 754                         continue;
 755                 sock_num = rte_lcore_to_socket_id(i);
 756                 if (new_socket_id(sock_num)) {
 757                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 758                                 rte_exit(EXIT_FAILURE,
 759                                          "Total sockets greater than %u\n",
 760                                          RTE_MAX_NUMA_NODES);
 761                         }
 762                         socket_ids[num_sockets++] = sock_num;
 763                 }
 764                 if (i == rte_get_main_lcore())
 765                         continue;
 766                 fwd_lcores_cpuids[nb_lc++] = i;
 767         }
 768         nb_lcores = (lcoreid_t) nb_lc;
 769         nb_cfg_lcores = nb_lcores;
 770         nb_fwd_lcores = 1;
 771 }
 772
 773 static void
 774 set_def_peer_eth_addrs(void)
 775 {
 776         portid_t i;
 777
 778         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 779                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 780                 peer_eth_addrs[i].addr_bytes[5] = i;
 781         }
 782 }
 783
 784 static void
 785 set_default_fwd_ports_config(void)
 786 {
 787         portid_t pt_id;
 788         int i = 0;
 789
 790         RTE_ETH_FOREACH_DEV(pt_id) {
 791                 fwd_ports_ids[i++] = pt_id;
 792
 793                 /* Update sockets info according to the attached device */
 794                 int socket_id = rte_eth_dev_socket_id(pt_id);
 795                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 796                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 797                                 rte_exit(EXIT_FAILURE,
 798                                          "Total sockets greater than %u\n",
 799                                          RTE_MAX_NUMA_NODES);
 800                         }
 801                         socket_ids[num_sockets++] = socket_id;
 802                 }
 803         }
 804
 805         nb_cfg_ports = nb_ports;
 806         nb_fwd_ports = nb_ports;
 807 }
 808
 809 void
 810 set_def_fwd_config(void)
 811 {
 812         set_default_fwd_lcores_config();
 813         set_def_peer_eth_addrs();
 814         set_default_fwd_ports_config();
 815 }
 816
 817 #ifndef RTE_EXEC_ENV_WINDOWS
 818 /* extremely pessimistic estimation of memory required to create a mempool */
 819 static int
 820 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 821 {
 822         unsigned int n_pages, mbuf_per_pg, leftover;
 823         uint64_t total_mem, mbuf_mem, obj_sz;
 824
 825         /* there is no good way to predict how much space the mempool will
 826          * occupy because it will allocate chunks on the fly, and some of those
 827          * will come from default DPDK memory while some will come from our
 828          * external memory, so just assume 128MB will be enough for everyone.
 829          */
 830         uint64_t hdr_mem = 128 << 20;
 831
 832         /* account for possible non-contiguousness */
 833         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 834         if (obj_sz > pgsz) {
 835                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 836                 return -1;
 837         }
 838
 839         mbuf_per_pg = pgsz / obj_sz;
 840         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 841         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 842
 843         mbuf_mem = n_pages * pgsz;
 844
 845         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 846
 847         if (total_mem > SIZE_MAX) {
 848                 TESTPMD_LOG(ERR, "Memory size too big\n");
 849                 return -1;
 850         }
 851         *out = (size_t)total_mem;
 852
 853         return 0;
 854 }
 855
 856 static int
 857 pagesz_flags(uint64_t page_sz)
 858 {
 859         /* as per mmap() manpage, all page sizes are log2 of page size
 860          * shifted by MAP_HUGE_SHIFT
 861          */
 862         int log2 = rte_log2_u64(page_sz);
 863
 864         return (log2 << HUGE_SHIFT);
 865 }
 866
 867 static void *
 868 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 869 {
 870         void *addr;
 871         int flags;
 872
 873         /* allocate anonymous hugepages */
 874         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 875         if (huge)
 876                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 877
 878         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 879         if (addr == MAP_FAILED)
 880                 return NULL;
 881
 882         return addr;
 883 }
 884
 885 struct extmem_param {
 886         void *addr;
 887         size_t len;
 888         size_t pgsz;
 889         rte_iova_t *iova_table;
 890         unsigned int iova_table_len;
 891 };
 892
 893 static int
 894 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 895                 bool huge)
 896 {
 897         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 898                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 899         unsigned int cur_page, n_pages, pgsz_idx;
 900         size_t mem_sz, cur_pgsz;
 901         rte_iova_t *iovas = NULL;
 902         void *addr;
 903         int ret;
 904
 905         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 906                 /* skip anything that is too big */
 907                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 908                         continue;
 909
 910                 cur_pgsz = pgsizes[pgsz_idx];
 911
 912                 /* if we were told not to allocate hugepages, override */
 913                 if (!huge)
 914                         cur_pgsz = sysconf(_SC_PAGESIZE);
 915
 916                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 917                 if (ret < 0) {
 918                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 919                         return -1;
 920                 }
 921
 922                 /* allocate our memory */
 923                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 924
 925                 /* if we couldn't allocate memory with a specified page size,
 926                  * that doesn't mean we can't do it with other page sizes, so
 927                  * try another one.
 928                  */
 929                 if (addr == NULL)
 930                         continue;
 931
 932                 /* store IOVA addresses for every page in this memory area */
 933                 n_pages = mem_sz / cur_pgsz;
 934
 935                 iovas = malloc(sizeof(*iovas) * n_pages);
 936
 937                 if (iovas == NULL) {
 938                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 939                         goto fail;
 940                 }
 941                 /* lock memory if it's not huge pages */
 942                 if (!huge)
 943                         mlock(addr, mem_sz);
 944
 945                 /* populate IOVA addresses */
 946                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 947                         rte_iova_t iova;
 948                         size_t offset;
 949                         void *cur;
 950
 951                         offset = cur_pgsz * cur_page;
 952                         cur = RTE_PTR_ADD(addr, offset);
 953
 954                         /* touch the page before getting its IOVA */
 955                         *(volatile char *)cur = 0;
 956
 957                         iova = rte_mem_virt2iova(cur);
 958
 959                         iovas[cur_page] = iova;
 960                 }
 961
 962                 break;
 963         }
 964         /* if we couldn't allocate anything */
 965         if (iovas == NULL)
 966                 return -1;
 967
 968         param->addr = addr;
 969         param->len = mem_sz;
 970         param->pgsz = cur_pgsz;
 971         param->iova_table = iovas;
 972         param->iova_table_len = n_pages;
 973
 974         return 0;
 975 fail:
 976         free(iovas);
 977         if (addr)
 978                 munmap(addr, mem_sz);
 979
 980         return -1;
 981 }
 982
 983 static int
 984 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 985 {
 986         struct extmem_param param;
 987         int socket_id, ret;
 988
 989         memset(&param, 0, sizeof(param));
 990
 991         /* check if our heap exists */
 992         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 993         if (socket_id < 0) {
 994                 /* create our heap */
 995                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 996                 if (ret < 0) {
 997                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 998                         return -1;
 999                 }
1000         }
1001
1002         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
1003         if (ret < 0) {
1004                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
1005                 return -1;
1006         }
1007
1008         /* we now have a valid memory area, so add it to heap */
1009         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
1010                         param.addr, param.len, param.iova_table,
1011                         param.iova_table_len, param.pgsz);
1012
1013         /* when using VFIO, memory is automatically mapped for DMA by EAL */
1014
1015         /* not needed any more */
1016         free(param.iova_table);
1017
1018         if (ret < 0) {
1019                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
1020                 munmap(param.addr, param.len);
1021                 return -1;
1022         }
1023
1024         /* success */
1025
1026         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
1027                         param.len >> 20);
1028
1029         return 0;
1030 }
1031 static void
1032 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1033              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1034 {
1035         uint16_t pid = 0;
1036         int ret;
1037
1038         RTE_ETH_FOREACH_DEV(pid) {
1039                 struct rte_eth_dev_info dev_info;
1040
1041                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1042                 if (ret != 0) {
1043                         TESTPMD_LOG(DEBUG,
1044                                     "unable to get device info for port %d on addr 0x%p,"
1045                                     "mempool unmapping will not be performed\n",
1046                                     pid, memhdr->addr);
1047                         continue;
1048                 }
1049
1050                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
1051                 if (ret) {
1052                         TESTPMD_LOG(DEBUG,
1053                                     "unable to DMA unmap addr 0x%p "
1054                                     "for device %s\n",
1055                                     memhdr->addr, dev_info.device->name);
1056                 }
1057         }
1058         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
1059         if (ret) {
1060                 TESTPMD_LOG(DEBUG,
1061                             "unable to un-register addr 0x%p\n", memhdr->addr);
1062         }
1063 }
1064
1065 static void
1066 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1067            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1068 {
1069         uint16_t pid = 0;
1070         size_t page_size = sysconf(_SC_PAGESIZE);
1071         int ret;
1072
1073         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1074                                   page_size);
1075         if (ret) {
1076                 TESTPMD_LOG(DEBUG,
1077                             "unable to register addr 0x%p\n", memhdr->addr);
1078                 return;
1079         }
1080         RTE_ETH_FOREACH_DEV(pid) {
1081                 struct rte_eth_dev_info dev_info;
1082
1083                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1084                 if (ret != 0) {
1085                         TESTPMD_LOG(DEBUG,
1086                                     "unable to get device info for port %d on addr 0x%p,"
1087                                     "mempool mapping will not be performed\n",
1088                                     pid, memhdr->addr);
1089                         continue;
1090                 }
1091                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1092                 if (ret) {
1093                         TESTPMD_LOG(DEBUG,
1094                                     "unable to DMA map addr 0x%p "
1095                                     "for device %s\n",
1096                                     memhdr->addr, dev_info.device->name);
1097                 }
1098         }
1099 }
1100 #endif
1101
1102 static unsigned int
1103 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1104             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1105 {
1106         struct rte_pktmbuf_extmem *xmem;
1107         unsigned int ext_num, zone_num, elt_num;
1108         uint16_t elt_size;
1109
1110         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1111         elt_num = EXTBUF_ZONE_SIZE / elt_size;
1112         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1113
1114         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1115         if (xmem == NULL) {
1116                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1117                                  "external buffer descriptors\n");
1118                 *ext_mem = NULL;
1119                 return 0;
1120         }
1121         for (ext_num = 0; ext_num < zone_num; ext_num++) {
1122                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1123                 const struct rte_memzone *mz;
1124                 char mz_name[RTE_MEMZONE_NAMESIZE];
1125                 int ret;
1126
1127                 ret = snprintf(mz_name, sizeof(mz_name),
1128                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1129                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1130                         errno = ENAMETOOLONG;
1131                         ext_num = 0;
1132                         break;
1133                 }
1134                 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE,
1135                                          socket_id,
1136                                          RTE_MEMZONE_IOVA_CONTIG |
1137                                          RTE_MEMZONE_1GB |
1138                                          RTE_MEMZONE_SIZE_HINT_ONLY);
1139                 if (mz == NULL) {
1140                         /*
1141                          * The caller exits on external buffer creation
1142                          * error, so there is no need to free memzones.
1143                          */
1144                         errno = ENOMEM;
1145                         ext_num = 0;
1146                         break;
1147                 }
1148                 xseg->buf_ptr = mz->addr;
1149                 xseg->buf_iova = mz->iova;
1150                 xseg->buf_len = EXTBUF_ZONE_SIZE;
1151                 xseg->elt_size = elt_size;
1152         }
1153         if (ext_num == 0 && xmem != NULL) {
1154                 free(xmem);
1155                 xmem = NULL;
1156         }
1157         *ext_mem = xmem;
1158         return ext_num;
1159 }
1160
1161 /*
1162  * Configuration initialisation done once at init time.
1163  */
1164 static struct rte_mempool *
1165 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1166                  unsigned int socket_id, uint16_t size_idx)
1167 {
1168         char pool_name[RTE_MEMPOOL_NAMESIZE];
1169         struct rte_mempool *rte_mp = NULL;
1170 #ifndef RTE_EXEC_ENV_WINDOWS
1171         uint32_t mb_size;
1172
1173         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1174 #endif
1175         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1176         if (!is_proc_primary()) {
1177                 rte_mp = rte_mempool_lookup(pool_name);
1178                 if (rte_mp == NULL)
1179                         rte_exit(EXIT_FAILURE,
1180                                 "Get mbuf pool for socket %u failed: %s\n",
1181                                 socket_id, rte_strerror(rte_errno));
1182                 return rte_mp;
1183         }
1184
1185         TESTPMD_LOG(INFO,
1186                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1187                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1188
1189         switch (mp_alloc_type) {
1190         case MP_ALLOC_NATIVE:
1191                 {
1192                         /* wrapper to rte_mempool_create() */
1193                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1194                                         rte_mbuf_best_mempool_ops());
1195                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1196                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1197                         break;
1198                 }
1199 #ifndef RTE_EXEC_ENV_WINDOWS
1200         case MP_ALLOC_ANON:
1201                 {
1202                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1203                                 mb_size, (unsigned int) mb_mempool_cache,
1204                                 sizeof(struct rte_pktmbuf_pool_private),
1205                                 socket_id, mempool_flags);
1206                         if (rte_mp == NULL)
1207                                 goto err;
1208
1209                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1210                                 rte_mempool_free(rte_mp);
1211                                 rte_mp = NULL;
1212                                 goto err;
1213                         }
1214                         rte_pktmbuf_pool_init(rte_mp, NULL);
1215                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1216                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1217                         break;
1218                 }
1219         case MP_ALLOC_XMEM:
1220         case MP_ALLOC_XMEM_HUGE:
1221                 {
1222                         int heap_socket;
1223                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1224
1225                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1226                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1227
1228                         heap_socket =
1229                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1230                         if (heap_socket < 0)
1231                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1232
1233                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1234                                         rte_mbuf_best_mempool_ops());
1235                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1236                                         mb_mempool_cache, 0, mbuf_seg_size,
1237                                         heap_socket);
1238                         break;
1239                 }
1240 #endif
1241         case MP_ALLOC_XBUF:
1242                 {
1243                         struct rte_pktmbuf_extmem *ext_mem;
1244                         unsigned int ext_num;
1245
1246                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1247                                                socket_id, pool_name, &ext_mem);
1248                         if (ext_num == 0)
1249                                 rte_exit(EXIT_FAILURE,
1250                                          "Can't create pinned data buffers\n");
1251
1252                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1253                                         rte_mbuf_best_mempool_ops());
1254                         rte_mp = rte_pktmbuf_pool_create_extbuf
1255                                         (pool_name, nb_mbuf, mb_mempool_cache,
1256                                          0, mbuf_seg_size, socket_id,
1257                                          ext_mem, ext_num);
1258                         free(ext_mem);
1259                         break;
1260                 }
1261         default:
1262                 {
1263                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1264                 }
1265         }
1266
1267 #ifndef RTE_EXEC_ENV_WINDOWS
1268 err:
1269 #endif
1270         if (rte_mp == NULL) {
1271                 rte_exit(EXIT_FAILURE,
1272                         "Creation of mbuf pool for socket %u failed: %s\n",
1273                         socket_id, rte_strerror(rte_errno));
1274         } else if (verbose_level > 0) {
1275                 rte_mempool_dump(stdout, rte_mp);
1276         }
1277         return rte_mp;
1278 }
1279
1280 /*
1281  * Check given socket id is valid or not with NUMA mode,
1282  * if valid, return 0, else return -1
1283  */
1284 static int
1285 check_socket_id(const unsigned int socket_id)
1286 {
1287         static int warning_once = 0;
1288
1289         if (new_socket_id(socket_id)) {
1290                 if (!warning_once && numa_support)
1291                         fprintf(stderr,
1292                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1293                 warning_once = 1;
1294                 return -1;
1295         }
1296         return 0;
1297 }
1298
1299 /*
1300  * Get the allowed maximum number of RX queues.
1301  * *pid return the port id which has minimal value of
1302  * max_rx_queues in all ports.
1303  */
1304 queueid_t
1305 get_allowed_max_nb_rxq(portid_t *pid)
1306 {
1307         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1308         bool max_rxq_valid = false;
1309         portid_t pi;
1310         struct rte_eth_dev_info dev_info;
1311
1312         RTE_ETH_FOREACH_DEV(pi) {
1313                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1314                         continue;
1315
1316                 max_rxq_valid = true;
1317                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1318                         allowed_max_rxq = dev_info.max_rx_queues;
1319                         *pid = pi;
1320                 }
1321         }
1322         return max_rxq_valid ? allowed_max_rxq : 0;
1323 }
1324
1325 /*
1326  * Check input rxq is valid or not.
1327  * If input rxq is not greater than any of maximum number
1328  * of RX queues of all ports, it is valid.
1329  * if valid, return 0, else return -1
1330  */
1331 int
1332 check_nb_rxq(queueid_t rxq)
1333 {
1334         queueid_t allowed_max_rxq;
1335         portid_t pid = 0;
1336
1337         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1338         if (rxq > allowed_max_rxq) {
1339                 fprintf(stderr,
1340                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1341                         rxq, allowed_max_rxq, pid);
1342                 return -1;
1343         }
1344         return 0;
1345 }
1346
1347 /*
1348  * Get the allowed maximum number of TX queues.
1349  * *pid return the port id which has minimal value of
1350  * max_tx_queues in all ports.
1351  */
1352 queueid_t
1353 get_allowed_max_nb_txq(portid_t *pid)
1354 {
1355         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1356         bool max_txq_valid = false;
1357         portid_t pi;
1358         struct rte_eth_dev_info dev_info;
1359
1360         RTE_ETH_FOREACH_DEV(pi) {
1361                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1362                         continue;
1363
1364                 max_txq_valid = true;
1365                 if (dev_info.max_tx_queues < allowed_max_txq) {
1366                         allowed_max_txq = dev_info.max_tx_queues;
1367                         *pid = pi;
1368                 }
1369         }
1370         return max_txq_valid ? allowed_max_txq : 0;
1371 }
1372
1373 /*
1374  * Check input txq is valid or not.
1375  * If input txq is not greater than any of maximum number
1376  * of TX queues of all ports, it is valid.
1377  * if valid, return 0, else return -1
1378  */
1379 int
1380 check_nb_txq(queueid_t txq)
1381 {
1382         queueid_t allowed_max_txq;
1383         portid_t pid = 0;
1384
1385         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1386         if (txq > allowed_max_txq) {
1387                 fprintf(stderr,
1388                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1389                         txq, allowed_max_txq, pid);
1390                 return -1;
1391         }
1392         return 0;
1393 }
1394
1395 /*
1396  * Get the allowed maximum number of RXDs of every rx queue.
1397  * *pid return the port id which has minimal value of
1398  * max_rxd in all queues of all ports.
1399  */
1400 static uint16_t
1401 get_allowed_max_nb_rxd(portid_t *pid)
1402 {
1403         uint16_t allowed_max_rxd = UINT16_MAX;
1404         portid_t pi;
1405         struct rte_eth_dev_info dev_info;
1406
1407         RTE_ETH_FOREACH_DEV(pi) {
1408                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1409                         continue;
1410
1411                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1412                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1413                         *pid = pi;
1414                 }
1415         }
1416         return allowed_max_rxd;
1417 }
1418
1419 /*
1420  * Get the allowed minimal number of RXDs of every rx queue.
1421  * *pid return the port id which has minimal value of
1422  * min_rxd in all queues of all ports.
1423  */
1424 static uint16_t
1425 get_allowed_min_nb_rxd(portid_t *pid)
1426 {
1427         uint16_t allowed_min_rxd = 0;
1428         portid_t pi;
1429         struct rte_eth_dev_info dev_info;
1430
1431         RTE_ETH_FOREACH_DEV(pi) {
1432                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1433                         continue;
1434
1435                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1436                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1437                         *pid = pi;
1438                 }
1439         }
1440
1441         return allowed_min_rxd;
1442 }
1443
1444 /*
1445  * Check input rxd is valid or not.
1446  * If input rxd is not greater than any of maximum number
1447  * of RXDs of every Rx queues and is not less than any of
1448  * minimal number of RXDs of every Rx queues, it is valid.
1449  * if valid, return 0, else return -1
1450  */
1451 int
1452 check_nb_rxd(queueid_t rxd)
1453 {
1454         uint16_t allowed_max_rxd;
1455         uint16_t allowed_min_rxd;
1456         portid_t pid = 0;
1457
1458         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1459         if (rxd > allowed_max_rxd) {
1460                 fprintf(stderr,
1461                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1462                         rxd, allowed_max_rxd, pid);
1463                 return -1;
1464         }
1465
1466         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1467         if (rxd < allowed_min_rxd) {
1468                 fprintf(stderr,
1469                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1470                         rxd, allowed_min_rxd, pid);
1471                 return -1;
1472         }
1473
1474         return 0;
1475 }
1476
1477 /*
1478  * Get the allowed maximum number of TXDs of every rx queues.
1479  * *pid return the port id which has minimal value of
1480  * max_txd in every tx queue.
1481  */
1482 static uint16_t
1483 get_allowed_max_nb_txd(portid_t *pid)
1484 {
1485         uint16_t allowed_max_txd = UINT16_MAX;
1486         portid_t pi;
1487         struct rte_eth_dev_info dev_info;
1488
1489         RTE_ETH_FOREACH_DEV(pi) {
1490                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1491                         continue;
1492
1493                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1494                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1495                         *pid = pi;
1496                 }
1497         }
1498         return allowed_max_txd;
1499 }
1500
1501 /*
1502  * Get the allowed maximum number of TXDs of every tx queues.
1503  * *pid return the port id which has minimal value of
1504  * min_txd in every tx queue.
1505  */
1506 static uint16_t
1507 get_allowed_min_nb_txd(portid_t *pid)
1508 {
1509         uint16_t allowed_min_txd = 0;
1510         portid_t pi;
1511         struct rte_eth_dev_info dev_info;
1512
1513         RTE_ETH_FOREACH_DEV(pi) {
1514                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1515                         continue;
1516
1517                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1518                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1519                         *pid = pi;
1520                 }
1521         }
1522
1523         return allowed_min_txd;
1524 }
1525
1526 /*
1527  * Check input txd is valid or not.
1528  * If input txd is not greater than any of maximum number
1529  * of TXDs of every Rx queues, it is valid.
1530  * if valid, return 0, else return -1
1531  */
1532 int
1533 check_nb_txd(queueid_t txd)
1534 {
1535         uint16_t allowed_max_txd;
1536         uint16_t allowed_min_txd;
1537         portid_t pid = 0;
1538
1539         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1540         if (txd > allowed_max_txd) {
1541                 fprintf(stderr,
1542                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1543                         txd, allowed_max_txd, pid);
1544                 return -1;
1545         }
1546
1547         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1548         if (txd < allowed_min_txd) {
1549                 fprintf(stderr,
1550                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1551                         txd, allowed_min_txd, pid);
1552                 return -1;
1553         }
1554         return 0;
1555 }
1556
1557
1558 /*
1559  * Get the allowed maximum number of hairpin queues.
1560  * *pid return the port id which has minimal value of
1561  * max_hairpin_queues in all ports.
1562  */
1563 queueid_t
1564 get_allowed_max_nb_hairpinq(portid_t *pid)
1565 {
1566         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1567         portid_t pi;
1568         struct rte_eth_hairpin_cap cap;
1569
1570         RTE_ETH_FOREACH_DEV(pi) {
1571                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1572                         *pid = pi;
1573                         return 0;
1574                 }
1575                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1576                         allowed_max_hairpinq = cap.max_nb_queues;
1577                         *pid = pi;
1578                 }
1579         }
1580         return allowed_max_hairpinq;
1581 }
1582
1583 /*
1584  * Check input hairpin is valid or not.
1585  * If input hairpin is not greater than any of maximum number
1586  * of hairpin queues of all ports, it is valid.
1587  * if valid, return 0, else return -1
1588  */
1589 int
1590 check_nb_hairpinq(queueid_t hairpinq)
1591 {
1592         queueid_t allowed_max_hairpinq;
1593         portid_t pid = 0;
1594
1595         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1596         if (hairpinq > allowed_max_hairpinq) {
1597                 fprintf(stderr,
1598                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1599                         hairpinq, allowed_max_hairpinq, pid);
1600                 return -1;
1601         }
1602         return 0;
1603 }
1604
1605 static int
1606 get_eth_overhead(struct rte_eth_dev_info *dev_info)
1607 {
1608         uint32_t eth_overhead;
1609
1610         if (dev_info->max_mtu != UINT16_MAX &&
1611             dev_info->max_rx_pktlen > dev_info->max_mtu)
1612                 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1613         else
1614                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1615
1616         return eth_overhead;
1617 }
1618
1619 static void
1620 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1621 {
1622         struct rte_port *port = &ports[pid];
1623         int ret;
1624         int i;
1625
1626         eth_rx_metadata_negotiate_mp(pid);
1627
1628         port->dev_conf.txmode = tx_mode;
1629         port->dev_conf.rxmode = rx_mode;
1630
1631         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1632         if (ret != 0)
1633                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1634
1635         if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1636                 port->dev_conf.txmode.offloads &=
1637                         ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1638
1639         /* Apply Rx offloads configuration */
1640         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1641                 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1642         /* Apply Tx offloads configuration */
1643         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1644                 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1645
1646         if (eth_link_speed)
1647                 port->dev_conf.link_speeds = eth_link_speed;
1648
1649         if (max_rx_pkt_len)
1650                 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1651                         get_eth_overhead(&port->dev_info);
1652
1653         /* set flag to initialize port/queue */
1654         port->need_reconfig = 1;
1655         port->need_reconfig_queues = 1;
1656         port->socket_id = socket_id;
1657         port->tx_metadata = 0;
1658
1659         /*
1660          * Check for maximum number of segments per MTU.
1661          * Accordingly update the mbuf data size.
1662          */
1663         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1664             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1665                 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1666                 uint16_t mtu;
1667
1668                 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1669                         uint16_t data_size = (mtu + eth_overhead) /
1670                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1671                         uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1672
1673                         if (buffer_size > mbuf_data_size[0]) {
1674                                 mbuf_data_size[0] = buffer_size;
1675                                 TESTPMD_LOG(WARNING,
1676                                         "Configured mbuf size of the first segment %hu\n",
1677                                         mbuf_data_size[0]);
1678                         }
1679                 }
1680         }
1681 }
1682
1683 static void
1684 init_config(void)
1685 {
1686         portid_t pid;
1687         struct rte_mempool *mbp;
1688         unsigned int nb_mbuf_per_pool;
1689         lcoreid_t  lc_id;
1690 #ifdef RTE_LIB_GRO
1691         struct rte_gro_param gro_param;
1692 #endif
1693 #ifdef RTE_LIB_GSO
1694         uint32_t gso_types;
1695 #endif
1696
1697         /* Configuration of logical cores. */
1698         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1699                                 sizeof(struct fwd_lcore *) * nb_lcores,
1700                                 RTE_CACHE_LINE_SIZE);
1701         if (fwd_lcores == NULL) {
1702                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1703                                                         "failed\n", nb_lcores);
1704         }
1705         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1706                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1707                                                sizeof(struct fwd_lcore),
1708                                                RTE_CACHE_LINE_SIZE);
1709                 if (fwd_lcores[lc_id] == NULL) {
1710                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1711                                                                 "failed\n");
1712                 }
1713                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1714         }
1715
1716         RTE_ETH_FOREACH_DEV(pid) {
1717                 uint32_t socket_id;
1718
1719                 if (numa_support) {
1720                         socket_id = port_numa[pid];
1721                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1722                                 socket_id = rte_eth_dev_socket_id(pid);
1723
1724                                 /*
1725                                  * if socket_id is invalid,
1726                                  * set to the first available socket.
1727                                  */
1728                                 if (check_socket_id(socket_id) < 0)
1729                                         socket_id = socket_ids[0];
1730                         }
1731                 } else {
1732                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1733                                     0 : socket_num;
1734                 }
1735                 /* Apply default TxRx configuration for all ports */
1736                 init_config_port_offloads(pid, socket_id);
1737         }
1738         /*
1739          * Create pools of mbuf.
1740          * If NUMA support is disabled, create a single pool of mbuf in
1741          * socket 0 memory by default.
1742          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1743          *
1744          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1745          * nb_txd can be configured at run time.
1746          */
1747         if (param_total_num_mbufs)
1748                 nb_mbuf_per_pool = param_total_num_mbufs;
1749         else {
1750                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1751                         (nb_lcores * mb_mempool_cache) +
1752                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1753                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1754         }
1755
1756         if (numa_support) {
1757                 uint8_t i, j;
1758
1759                 for (i = 0; i < num_sockets; i++)
1760                         for (j = 0; j < mbuf_data_size_n; j++)
1761                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1762                                         mbuf_pool_create(mbuf_data_size[j],
1763                                                           nb_mbuf_per_pool,
1764                                                           socket_ids[i], j);
1765         } else {
1766                 uint8_t i;
1767
1768                 for (i = 0; i < mbuf_data_size_n; i++)
1769                         mempools[i] = mbuf_pool_create
1770                                         (mbuf_data_size[i],
1771                                          nb_mbuf_per_pool,
1772                                          socket_num == UMA_NO_CONFIG ?
1773                                          0 : socket_num, i);
1774         }
1775
1776         init_port_config();
1777
1778 #ifdef RTE_LIB_GSO
1779         gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1780                 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1781 #endif
1782         /*
1783          * Records which Mbuf pool to use by each logical core, if needed.
1784          */
1785         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1786                 mbp = mbuf_pool_find(
1787                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1788
1789                 if (mbp == NULL)
1790                         mbp = mbuf_pool_find(0, 0);
1791                 fwd_lcores[lc_id]->mbp = mbp;
1792 #ifdef RTE_LIB_GSO
1793                 /* initialize GSO context */
1794                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1795                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1796                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1797                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1798                         RTE_ETHER_CRC_LEN;
1799                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1800 #endif
1801         }
1802
1803         fwd_config_setup();
1804
1805 #ifdef RTE_LIB_GRO
1806         /* create a gro context for each lcore */
1807         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1808         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1809         gro_param.max_item_per_flow = MAX_PKT_BURST;
1810         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1811                 gro_param.socket_id = rte_lcore_to_socket_id(
1812                                 fwd_lcores_cpuids[lc_id]);
1813                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1814                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1815                         rte_exit(EXIT_FAILURE,
1816                                         "rte_gro_ctx_create() failed\n");
1817                 }
1818         }
1819 #endif
1820 }
1821
1822
1823 void
1824 reconfig(portid_t new_port_id, unsigned socket_id)
1825 {
1826         /* Reconfiguration of Ethernet ports. */
1827         init_config_port_offloads(new_port_id, socket_id);
1828         init_port_config();
1829 }
1830
1831
1832 int
1833 init_fwd_streams(void)
1834 {
1835         portid_t pid;
1836         struct rte_port *port;
1837         streamid_t sm_id, nb_fwd_streams_new;
1838         queueid_t q;
1839
1840         /* set socket id according to numa or not */
1841         RTE_ETH_FOREACH_DEV(pid) {
1842                 port = &ports[pid];
1843                 if (nb_rxq > port->dev_info.max_rx_queues) {
1844                         fprintf(stderr,
1845                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1846                                 nb_rxq, port->dev_info.max_rx_queues);
1847                         return -1;
1848                 }
1849                 if (nb_txq > port->dev_info.max_tx_queues) {
1850                         fprintf(stderr,
1851                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1852                                 nb_txq, port->dev_info.max_tx_queues);
1853                         return -1;
1854                 }
1855                 if (numa_support) {
1856                         if (port_numa[pid] != NUMA_NO_CONFIG)
1857                                 port->socket_id = port_numa[pid];
1858                         else {
1859                                 port->socket_id = rte_eth_dev_socket_id(pid);
1860
1861                                 /*
1862                                  * if socket_id is invalid,
1863                                  * set to the first available socket.
1864                                  */
1865                                 if (check_socket_id(port->socket_id) < 0)
1866                                         port->socket_id = socket_ids[0];
1867                         }
1868                 }
1869                 else {
1870                         if (socket_num == UMA_NO_CONFIG)
1871                                 port->socket_id = 0;
1872                         else
1873                                 port->socket_id = socket_num;
1874                 }
1875         }
1876
1877         q = RTE_MAX(nb_rxq, nb_txq);
1878         if (q == 0) {
1879                 fprintf(stderr,
1880                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1881                 return -1;
1882         }
1883         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1884         if (nb_fwd_streams_new == nb_fwd_streams)
1885                 return 0;
1886         /* clear the old */
1887         if (fwd_streams != NULL) {
1888                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1889                         if (fwd_streams[sm_id] == NULL)
1890                                 continue;
1891                         rte_free(fwd_streams[sm_id]);
1892                         fwd_streams[sm_id] = NULL;
1893                 }
1894                 rte_free(fwd_streams);
1895                 fwd_streams = NULL;
1896         }
1897
1898         /* init new */
1899         nb_fwd_streams = nb_fwd_streams_new;
1900         if (nb_fwd_streams) {
1901                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1902                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1903                         RTE_CACHE_LINE_SIZE);
1904                 if (fwd_streams == NULL)
1905                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1906                                  " (struct fwd_stream *)) failed\n",
1907                                  nb_fwd_streams);
1908
1909                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1910                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1911                                 " struct fwd_stream", sizeof(struct fwd_stream),
1912                                 RTE_CACHE_LINE_SIZE);
1913                         if (fwd_streams[sm_id] == NULL)
1914                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1915                                          "(struct fwd_stream) failed\n");
1916                 }
1917         }
1918
1919         return 0;
1920 }
1921
1922 static void
1923 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1924 {
1925         uint64_t total_burst, sburst;
1926         uint64_t nb_burst;
1927         uint64_t burst_stats[4];
1928         uint16_t pktnb_stats[4];
1929         uint16_t nb_pkt;
1930         int burst_percent[4], sburstp;
1931         int i;
1932
1933         /*
1934          * First compute the total number of packet bursts and the
1935          * two highest numbers of bursts of the same number of packets.
1936          */
1937         memset(&burst_stats, 0x0, sizeof(burst_stats));
1938         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1939
1940         /* Show stats for 0 burst size always */
1941         total_burst = pbs->pkt_burst_spread[0];
1942         burst_stats[0] = pbs->pkt_burst_spread[0];
1943         pktnb_stats[0] = 0;
1944
1945         /* Find the next 2 burst sizes with highest occurrences. */
1946         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1947                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1948
1949                 if (nb_burst == 0)
1950                         continue;
1951
1952                 total_burst += nb_burst;
1953
1954                 if (nb_burst > burst_stats[1]) {
1955                         burst_stats[2] = burst_stats[1];
1956                         pktnb_stats[2] = pktnb_stats[1];
1957                         burst_stats[1] = nb_burst;
1958                         pktnb_stats[1] = nb_pkt;
1959                 } else if (nb_burst > burst_stats[2]) {
1960                         burst_stats[2] = nb_burst;
1961                         pktnb_stats[2] = nb_pkt;
1962                 }
1963         }
1964         if (total_burst == 0)
1965                 return;
1966
1967         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1968         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1969                 if (i == 3) {
1970                         printf("%d%% of other]\n", 100 - sburstp);
1971                         return;
1972                 }
1973
1974                 sburst += burst_stats[i];
1975                 if (sburst == total_burst) {
1976                         printf("%d%% of %d pkts]\n",
1977                                 100 - sburstp, (int) pktnb_stats[i]);
1978                         return;
1979                 }
1980
1981                 burst_percent[i] =
1982                         (double)burst_stats[i] / total_burst * 100;
1983                 printf("%d%% of %d pkts + ",
1984                         burst_percent[i], (int) pktnb_stats[i]);
1985                 sburstp += burst_percent[i];
1986         }
1987 }
1988
1989 static void
1990 fwd_stream_stats_display(streamid_t stream_id)
1991 {
1992         struct fwd_stream *fs;
1993         static const char *fwd_top_stats_border = "-------";
1994
1995         fs = fwd_streams[stream_id];
1996         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1997             (fs->fwd_dropped == 0))
1998                 return;
1999         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
2000                "TX Port=%2d/Queue=%2d %s\n",
2001                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
2002                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
2003         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
2004                " TX-dropped: %-14"PRIu64,
2005                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
2006
2007         /* if checksum mode */
2008         if (cur_fwd_eng == &csum_fwd_engine) {
2009                 printf("  RX- bad IP checksum: %-14"PRIu64
2010                        "  Rx- bad L4 checksum: %-14"PRIu64
2011                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
2012                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
2013                         fs->rx_bad_outer_l4_csum);
2014                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
2015                         fs->rx_bad_outer_ip_csum);
2016         } else {
2017                 printf("\n");
2018         }
2019
2020         if (record_burst_stats) {
2021                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
2022                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
2023         }
2024 }
2025
2026 void
2027 fwd_stats_display(void)
2028 {
2029         static const char *fwd_stats_border = "----------------------";
2030         static const char *acc_stats_border = "+++++++++++++++";
2031         struct {
2032                 struct fwd_stream *rx_stream;
2033                 struct fwd_stream *tx_stream;
2034                 uint64_t tx_dropped;
2035                 uint64_t rx_bad_ip_csum;
2036                 uint64_t rx_bad_l4_csum;
2037                 uint64_t rx_bad_outer_l4_csum;
2038                 uint64_t rx_bad_outer_ip_csum;
2039         } ports_stats[RTE_MAX_ETHPORTS];
2040         uint64_t total_rx_dropped = 0;
2041         uint64_t total_tx_dropped = 0;
2042         uint64_t total_rx_nombuf = 0;
2043         struct rte_eth_stats stats;
2044         uint64_t fwd_cycles = 0;
2045         uint64_t total_recv = 0;
2046         uint64_t total_xmit = 0;
2047         struct rte_port *port;
2048         streamid_t sm_id;
2049         portid_t pt_id;
2050         int ret;
2051         int i;
2052
2053         memset(ports_stats, 0, sizeof(ports_stats));
2054
2055         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2056                 struct fwd_stream *fs = fwd_streams[sm_id];
2057
2058                 if (cur_fwd_config.nb_fwd_streams >
2059                     cur_fwd_config.nb_fwd_ports) {
2060                         fwd_stream_stats_display(sm_id);
2061                 } else {
2062                         ports_stats[fs->tx_port].tx_stream = fs;
2063                         ports_stats[fs->rx_port].rx_stream = fs;
2064                 }
2065
2066                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
2067
2068                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
2069                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2070                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2071                                 fs->rx_bad_outer_l4_csum;
2072                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2073                                 fs->rx_bad_outer_ip_csum;
2074
2075                 if (record_core_cycles)
2076                         fwd_cycles += fs->core_cycles;
2077         }
2078         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2079                 pt_id = fwd_ports_ids[i];
2080                 port = &ports[pt_id];
2081
2082                 ret = rte_eth_stats_get(pt_id, &stats);
2083                 if (ret != 0) {
2084                         fprintf(stderr,
2085                                 "%s: Error: failed to get stats (port %u): %d",
2086                                 __func__, pt_id, ret);
2087                         continue;
2088                 }
2089                 stats.ipackets -= port->stats.ipackets;
2090                 stats.opackets -= port->stats.opackets;
2091                 stats.ibytes -= port->stats.ibytes;
2092                 stats.obytes -= port->stats.obytes;
2093                 stats.imissed -= port->stats.imissed;
2094                 stats.oerrors -= port->stats.oerrors;
2095                 stats.rx_nombuf -= port->stats.rx_nombuf;
2096
2097                 total_recv += stats.ipackets;
2098                 total_xmit += stats.opackets;
2099                 total_rx_dropped += stats.imissed;
2100                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2101                 total_tx_dropped += stats.oerrors;
2102                 total_rx_nombuf  += stats.rx_nombuf;
2103
2104                 printf("\n  %s Forward statistics for port %-2d %s\n",
2105                        fwd_stats_border, pt_id, fwd_stats_border);
2106
2107                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2108                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2109                        stats.ipackets + stats.imissed);
2110
2111                 if (cur_fwd_eng == &csum_fwd_engine) {
2112                         printf("  Bad-ipcsum: %-14"PRIu64
2113                                " Bad-l4csum: %-14"PRIu64
2114                                "Bad-outer-l4csum: %-14"PRIu64"\n",
2115                                ports_stats[pt_id].rx_bad_ip_csum,
2116                                ports_stats[pt_id].rx_bad_l4_csum,
2117                                ports_stats[pt_id].rx_bad_outer_l4_csum);
2118                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
2119                                ports_stats[pt_id].rx_bad_outer_ip_csum);
2120                 }
2121                 if (stats.ierrors + stats.rx_nombuf > 0) {
2122                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
2123                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2124                 }
2125
2126                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2127                        "TX-total: %-"PRIu64"\n",
2128                        stats.opackets, ports_stats[pt_id].tx_dropped,
2129                        stats.opackets + ports_stats[pt_id].tx_dropped);
2130
2131                 if (record_burst_stats) {
2132                         if (ports_stats[pt_id].rx_stream)
2133                                 pkt_burst_stats_display("RX",
2134                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
2135                         if (ports_stats[pt_id].tx_stream)
2136                                 pkt_burst_stats_display("TX",
2137                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2138                 }
2139
2140                 printf("  %s--------------------------------%s\n",
2141                        fwd_stats_border, fwd_stats_border);
2142         }
2143
2144         printf("\n  %s Accumulated forward statistics for all ports"
2145                "%s\n",
2146                acc_stats_border, acc_stats_border);
2147         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2148                "%-"PRIu64"\n"
2149                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2150                "%-"PRIu64"\n",
2151                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2152                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2153         if (total_rx_nombuf > 0)
2154                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2155         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
2156                "%s\n",
2157                acc_stats_border, acc_stats_border);
2158         if (record_core_cycles) {
2159 #define CYC_PER_MHZ 1E6
2160                 if (total_recv > 0 || total_xmit > 0) {
2161                         uint64_t total_pkts = 0;
2162                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2163                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2164                                 total_pkts = total_xmit;
2165                         else
2166                                 total_pkts = total_recv;
2167
2168                         printf("\n  CPU cycles/packet=%.2F (total cycles="
2169                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2170                                " MHz Clock\n",
2171                                (double) fwd_cycles / total_pkts,
2172                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2173                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2174                 }
2175         }
2176 }
2177
2178 void
2179 fwd_stats_reset(void)
2180 {
2181         streamid_t sm_id;
2182         portid_t pt_id;
2183         int ret;
2184         int i;
2185
2186         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2187                 pt_id = fwd_ports_ids[i];
2188                 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2189                 if (ret != 0)
2190                         fprintf(stderr,
2191                                 "%s: Error: failed to clear stats (port %u):%d",
2192                                 __func__, pt_id, ret);
2193         }
2194         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2195                 struct fwd_stream *fs = fwd_streams[sm_id];
2196
2197                 fs->rx_packets = 0;
2198                 fs->tx_packets = 0;
2199                 fs->fwd_dropped = 0;
2200                 fs->rx_bad_ip_csum = 0;
2201                 fs->rx_bad_l4_csum = 0;
2202                 fs->rx_bad_outer_l4_csum = 0;
2203                 fs->rx_bad_outer_ip_csum = 0;
2204
2205                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2206                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2207                 fs->core_cycles = 0;
2208         }
2209 }
2210
2211 static void
2212 flush_fwd_rx_queues(void)
2213 {
2214         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2215         portid_t  rxp;
2216         portid_t port_id;
2217         queueid_t rxq;
2218         uint16_t  nb_rx;
2219         uint16_t  i;
2220         uint8_t   j;
2221         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2222         uint64_t timer_period;
2223
2224         if (num_procs > 1) {
2225                 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2226                 return;
2227         }
2228
2229         /* convert to number of cycles */
2230         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2231
2232         for (j = 0; j < 2; j++) {
2233                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2234                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2235                                 port_id = fwd_ports_ids[rxp];
2236                                 /**
2237                                 * testpmd can stuck in the below do while loop
2238                                 * if rte_eth_rx_burst() always returns nonzero
2239                                 * packets. So timer is added to exit this loop
2240                                 * after 1sec timer expiry.
2241                                 */
2242                                 prev_tsc = rte_rdtsc();
2243                                 do {
2244                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2245                                                 pkts_burst, MAX_PKT_BURST);
2246                                         for (i = 0; i < nb_rx; i++)
2247                                                 rte_pktmbuf_free(pkts_burst[i]);
2248
2249                                         cur_tsc = rte_rdtsc();
2250                                         diff_tsc = cur_tsc - prev_tsc;
2251                                         timer_tsc += diff_tsc;
2252                                 } while ((nb_rx > 0) &&
2253                                         (timer_tsc < timer_period));
2254                                 timer_tsc = 0;
2255                         }
2256                 }
2257                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2258         }
2259 }
2260
2261 static void
2262 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2263 {
2264         struct fwd_stream **fsm;
2265         streamid_t nb_fs;
2266         streamid_t sm_id;
2267 #ifdef RTE_LIB_BITRATESTATS
2268         uint64_t tics_per_1sec;
2269         uint64_t tics_datum;
2270         uint64_t tics_current;
2271         uint16_t i, cnt_ports;
2272
2273         cnt_ports = nb_ports;
2274         tics_datum = rte_rdtsc();
2275         tics_per_1sec = rte_get_timer_hz();
2276 #endif
2277         fsm = &fwd_streams[fc->stream_idx];
2278         nb_fs = fc->stream_nb;
2279         do {
2280                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2281                         (*pkt_fwd)(fsm[sm_id]);
2282 #ifdef RTE_LIB_BITRATESTATS
2283                 if (bitrate_enabled != 0 &&
2284                                 bitrate_lcore_id == rte_lcore_id()) {
2285                         tics_current = rte_rdtsc();
2286                         if (tics_current - tics_datum >= tics_per_1sec) {
2287                                 /* Periodic bitrate calculation */
2288                                 for (i = 0; i < cnt_ports; i++)
2289                                         rte_stats_bitrate_calc(bitrate_data,
2290                                                 ports_ids[i]);
2291                                 tics_datum = tics_current;
2292                         }
2293                 }
2294 #endif
2295 #ifdef RTE_LIB_LATENCYSTATS
2296                 if (latencystats_enabled != 0 &&
2297                                 latencystats_lcore_id == rte_lcore_id())
2298                         rte_latencystats_update();
2299 #endif
2300
2301         } while (! fc->stopped);
2302 }
2303
2304 static int
2305 start_pkt_forward_on_core(void *fwd_arg)
2306 {
2307         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2308                              cur_fwd_config.fwd_eng->packet_fwd);
2309         return 0;
2310 }
2311
2312 /*
2313  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2314  * Used to start communication flows in network loopback test configurations.
2315  */
2316 static int
2317 run_one_txonly_burst_on_core(void *fwd_arg)
2318 {
2319         struct fwd_lcore *fwd_lc;
2320         struct fwd_lcore tmp_lcore;
2321
2322         fwd_lc = (struct fwd_lcore *) fwd_arg;
2323         tmp_lcore = *fwd_lc;
2324         tmp_lcore.stopped = 1;
2325         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2326         return 0;
2327 }
2328
2329 /*
2330  * Launch packet forwarding:
2331  *     - Setup per-port forwarding context.
2332  *     - launch logical cores with their forwarding configuration.
2333  */
2334 static void
2335 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2336 {
2337         unsigned int i;
2338         unsigned int lc_id;
2339         int diag;
2340
2341         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2342                 lc_id = fwd_lcores_cpuids[i];
2343                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2344                         fwd_lcores[i]->stopped = 0;
2345                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2346                                                      fwd_lcores[i], lc_id);
2347                         if (diag != 0)
2348                                 fprintf(stderr,
2349                                         "launch lcore %u failed - diag=%d\n",
2350                                         lc_id, diag);
2351                 }
2352         }
2353 }
2354
2355 /*
2356  * Launch packet forwarding configuration.
2357  */
2358 void
2359 start_packet_forwarding(int with_tx_first)
2360 {
2361         port_fwd_begin_t port_fwd_begin;
2362         port_fwd_end_t  port_fwd_end;
2363         unsigned int i;
2364
2365         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2366                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2367
2368         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2369                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2370
2371         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2372                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2373                 (!nb_rxq || !nb_txq))
2374                 rte_exit(EXIT_FAILURE,
2375                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2376                         cur_fwd_eng->fwd_mode_name);
2377
2378         if (all_ports_started() == 0) {
2379                 fprintf(stderr, "Not all ports were started\n");
2380                 return;
2381         }
2382         if (test_done == 0) {
2383                 fprintf(stderr, "Packet forwarding already started\n");
2384                 return;
2385         }
2386
2387         fwd_config_setup();
2388
2389         pkt_fwd_config_display(&cur_fwd_config);
2390         if (!pkt_fwd_shared_rxq_check())
2391                 return;
2392
2393         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2394         if (port_fwd_begin != NULL) {
2395                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2396                         if (port_fwd_begin(fwd_ports_ids[i])) {
2397                                 fprintf(stderr,
2398                                         "Packet forwarding is not ready\n");
2399                                 return;
2400                         }
2401                 }
2402         }
2403
2404         if (with_tx_first) {
2405                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2406                 if (port_fwd_begin != NULL) {
2407                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2408                                 if (port_fwd_begin(fwd_ports_ids[i])) {
2409                                         fprintf(stderr,
2410                                                 "Packet forwarding is not ready\n");
2411                                         return;
2412                                 }
2413                         }
2414                 }
2415         }
2416
2417         test_done = 0;
2418
2419         if(!no_flush_rx)
2420                 flush_fwd_rx_queues();
2421
2422         rxtx_config_display();
2423
2424         fwd_stats_reset();
2425         if (with_tx_first) {
2426                 while (with_tx_first--) {
2427                         launch_packet_forwarding(
2428                                         run_one_txonly_burst_on_core);
2429                         rte_eal_mp_wait_lcore();
2430                 }
2431                 port_fwd_end = tx_only_engine.port_fwd_end;
2432                 if (port_fwd_end != NULL) {
2433                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2434                                 (*port_fwd_end)(fwd_ports_ids[i]);
2435                 }
2436         }
2437         launch_packet_forwarding(start_pkt_forward_on_core);
2438 }
2439
2440 void
2441 stop_packet_forwarding(void)
2442 {
2443         port_fwd_end_t port_fwd_end;
2444         lcoreid_t lc_id;
2445         portid_t pt_id;
2446         int i;
2447
2448         if (test_done) {
2449                 fprintf(stderr, "Packet forwarding not started\n");
2450                 return;
2451         }
2452         printf("Telling cores to stop...");
2453         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2454                 fwd_lcores[lc_id]->stopped = 1;
2455         printf("\nWaiting for lcores to finish...\n");
2456         rte_eal_mp_wait_lcore();
2457         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2458         if (port_fwd_end != NULL) {
2459                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2460                         pt_id = fwd_ports_ids[i];
2461                         (*port_fwd_end)(pt_id);
2462                 }
2463         }
2464
2465         fwd_stats_display();
2466
2467         printf("\nDone.\n");
2468         test_done = 1;
2469 }
2470
2471 void
2472 dev_set_link_up(portid_t pid)
2473 {
2474         if (rte_eth_dev_set_link_up(pid) < 0)
2475                 fprintf(stderr, "\nSet link up fail.\n");
2476 }
2477
2478 void
2479 dev_set_link_down(portid_t pid)
2480 {
2481         if (rte_eth_dev_set_link_down(pid) < 0)
2482                 fprintf(stderr, "\nSet link down fail.\n");
2483 }
2484
2485 static int
2486 all_ports_started(void)
2487 {
2488         portid_t pi;
2489         struct rte_port *port;
2490
2491         RTE_ETH_FOREACH_DEV(pi) {
2492                 port = &ports[pi];
2493                 /* Check if there is a port which is not started */
2494                 if ((port->port_status != RTE_PORT_STARTED) &&
2495                         (port->slave_flag == 0))
2496                         return 0;
2497         }
2498
2499         /* No port is not started */
2500         return 1;
2501 }
2502
2503 int
2504 port_is_stopped(portid_t port_id)
2505 {
2506         struct rte_port *port = &ports[port_id];
2507
2508         if ((port->port_status != RTE_PORT_STOPPED) &&
2509             (port->slave_flag == 0))
2510                 return 0;
2511         return 1;
2512 }
2513
2514 int
2515 all_ports_stopped(void)
2516 {
2517         portid_t pi;
2518
2519         RTE_ETH_FOREACH_DEV(pi) {
2520                 if (!port_is_stopped(pi))
2521                         return 0;
2522         }
2523
2524         return 1;
2525 }
2526
2527 int
2528 port_is_started(portid_t port_id)
2529 {
2530         if (port_id_is_invalid(port_id, ENABLED_WARN))
2531                 return 0;
2532
2533         if (ports[port_id].port_status != RTE_PORT_STARTED)
2534                 return 0;
2535
2536         return 1;
2537 }
2538
2539 /* Configure the Rx and Tx hairpin queues for the selected port. */
2540 static int
2541 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2542 {
2543         queueid_t qi;
2544         struct rte_eth_hairpin_conf hairpin_conf = {
2545                 .peer_count = 1,
2546         };
2547         int i;
2548         int diag;
2549         struct rte_port *port = &ports[pi];
2550         uint16_t peer_rx_port = pi;
2551         uint16_t peer_tx_port = pi;
2552         uint32_t manual = 1;
2553         uint32_t tx_exp = hairpin_mode & 0x10;
2554
2555         if (!(hairpin_mode & 0xf)) {
2556                 peer_rx_port = pi;
2557                 peer_tx_port = pi;
2558                 manual = 0;
2559         } else if (hairpin_mode & 0x1) {
2560                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2561                                                        RTE_ETH_DEV_NO_OWNER);
2562                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2563                         peer_tx_port = rte_eth_find_next_owned_by(0,
2564                                                 RTE_ETH_DEV_NO_OWNER);
2565                 if (p_pi != RTE_MAX_ETHPORTS) {
2566                         peer_rx_port = p_pi;
2567                 } else {
2568                         uint16_t next_pi;
2569
2570                         /* Last port will be the peer RX port of the first. */
2571                         RTE_ETH_FOREACH_DEV(next_pi)
2572                                 peer_rx_port = next_pi;
2573                 }
2574                 manual = 1;
2575         } else if (hairpin_mode & 0x2) {
2576                 if (cnt_pi & 0x1) {
2577                         peer_rx_port = p_pi;
2578                 } else {
2579                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2580                                                 RTE_ETH_DEV_NO_OWNER);
2581                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2582                                 peer_rx_port = pi;
2583                 }
2584                 peer_tx_port = peer_rx_port;
2585                 manual = 1;
2586         }
2587
2588         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2589                 hairpin_conf.peers[0].port = peer_rx_port;
2590                 hairpin_conf.peers[0].queue = i + nb_rxq;
2591                 hairpin_conf.manual_bind = !!manual;
2592                 hairpin_conf.tx_explicit = !!tx_exp;
2593                 diag = rte_eth_tx_hairpin_queue_setup
2594                         (pi, qi, nb_txd, &hairpin_conf);
2595                 i++;
2596                 if (diag == 0)
2597                         continue;
2598
2599                 /* Fail to setup rx queue, return */
2600                 if (port->port_status == RTE_PORT_HANDLING)
2601                         port->port_status = RTE_PORT_STOPPED;
2602                 else
2603                         fprintf(stderr,
2604                                 "Port %d can not be set back to stopped\n", pi);
2605                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2606                         pi);
2607                 /* try to reconfigure queues next time */
2608                 port->need_reconfig_queues = 1;
2609                 return -1;
2610         }
2611         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2612                 hairpin_conf.peers[0].port = peer_tx_port;
2613                 hairpin_conf.peers[0].queue = i + nb_txq;
2614                 hairpin_conf.manual_bind = !!manual;
2615                 hairpin_conf.tx_explicit = !!tx_exp;
2616                 diag = rte_eth_rx_hairpin_queue_setup
2617                         (pi, qi, nb_rxd, &hairpin_conf);
2618                 i++;
2619                 if (diag == 0)
2620                         continue;
2621
2622                 /* Fail to setup rx queue, return */
2623                 if (port->port_status == RTE_PORT_HANDLING)
2624                         port->port_status = RTE_PORT_STOPPED;
2625                 else
2626                         fprintf(stderr,
2627                                 "Port %d can not be set back to stopped\n", pi);
2628                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2629                         pi);
2630                 /* try to reconfigure queues next time */
2631                 port->need_reconfig_queues = 1;
2632                 return -1;
2633         }
2634         return 0;
2635 }
2636
2637 /* Configure the Rx with optional split. */
2638 int
2639 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2640                uint16_t nb_rx_desc, unsigned int socket_id,
2641                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2642 {
2643         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2644         unsigned int i, mp_n;
2645         int ret;
2646
2647         if (rx_pkt_nb_segs <= 1 ||
2648             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2649                 rx_conf->rx_seg = NULL;
2650                 rx_conf->rx_nseg = 0;
2651                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2652                                              nb_rx_desc, socket_id,
2653                                              rx_conf, mp);
2654                 return ret;
2655         }
2656         for (i = 0; i < rx_pkt_nb_segs; i++) {
2657                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2658                 struct rte_mempool *mpx;
2659                 /*
2660                  * Use last valid pool for the segments with number
2661                  * exceeding the pool index.
2662                  */
2663                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2664                 mpx = mbuf_pool_find(socket_id, mp_n);
2665                 /* Handle zero as mbuf data buffer size. */
2666                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2667                                    rx_pkt_seg_lengths[i] :
2668                                    mbuf_data_size[mp_n];
2669                 rx_seg->offset = i < rx_pkt_nb_offs ?
2670                                    rx_pkt_seg_offsets[i] : 0;
2671                 rx_seg->mp = mpx ? mpx : mp;
2672         }
2673         rx_conf->rx_nseg = rx_pkt_nb_segs;
2674         rx_conf->rx_seg = rx_useg;
2675         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2676                                     socket_id, rx_conf, NULL);
2677         rx_conf->rx_seg = NULL;
2678         rx_conf->rx_nseg = 0;
2679         return ret;
2680 }
2681
2682 static int
2683 alloc_xstats_display_info(portid_t pi)
2684 {
2685         uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2686         uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2687         uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2688
2689         if (xstats_display_num == 0)
2690                 return 0;
2691
2692         *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2693         if (*ids_supp == NULL)
2694                 goto fail_ids_supp;
2695
2696         *prev_values = calloc(xstats_display_num,
2697                               sizeof(**prev_values));
2698         if (*prev_values == NULL)
2699                 goto fail_prev_values;
2700
2701         *curr_values = calloc(xstats_display_num,
2702                               sizeof(**curr_values));
2703         if (*curr_values == NULL)
2704                 goto fail_curr_values;
2705
2706         ports[pi].xstats_info.allocated = true;
2707
2708         return 0;
2709
2710 fail_curr_values:
2711         free(*prev_values);
2712 fail_prev_values:
2713         free(*ids_supp);
2714 fail_ids_supp:
2715         return -ENOMEM;
2716 }
2717
2718 static void
2719 free_xstats_display_info(portid_t pi)
2720 {
2721         if (!ports[pi].xstats_info.allocated)
2722                 return;
2723         free(ports[pi].xstats_info.ids_supp);
2724         free(ports[pi].xstats_info.prev_values);
2725         free(ports[pi].xstats_info.curr_values);
2726         ports[pi].xstats_info.allocated = false;
2727 }
2728
2729 /** Fill helper structures for specified port to show extended statistics. */
2730 static void
2731 fill_xstats_display_info_for_port(portid_t pi)
2732 {
2733         unsigned int stat, stat_supp;
2734         const char *xstat_name;
2735         struct rte_port *port;
2736         uint64_t *ids_supp;
2737         int rc;
2738
2739         if (xstats_display_num == 0)
2740                 return;
2741
2742         if (pi == (portid_t)RTE_PORT_ALL) {
2743                 fill_xstats_display_info();
2744                 return;
2745         }
2746
2747         port = &ports[pi];
2748         if (port->port_status != RTE_PORT_STARTED)
2749                 return;
2750
2751         if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2752                 rte_exit(EXIT_FAILURE,
2753                          "Failed to allocate xstats display memory\n");
2754
2755         ids_supp = port->xstats_info.ids_supp;
2756         for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2757                 xstat_name = xstats_display[stat].name;
2758                 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2759                                                    ids_supp + stat_supp);
2760                 if (rc != 0) {
2761                         fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2762                                 xstat_name, pi, stat);
2763                         continue;
2764                 }
2765                 stat_supp++;
2766         }
2767
2768         port->xstats_info.ids_supp_sz = stat_supp;
2769 }
2770
2771 /** Fill helper structures for all ports to show extended statistics. */
2772 static void
2773 fill_xstats_display_info(void)
2774 {
2775         portid_t pi;
2776
2777         if (xstats_display_num == 0)
2778                 return;
2779
2780         RTE_ETH_FOREACH_DEV(pi)
2781                 fill_xstats_display_info_for_port(pi);
2782 }
2783
2784 int
2785 start_port(portid_t pid)
2786 {
2787         int diag, need_check_link_status = -1;
2788         portid_t pi;
2789         portid_t p_pi = RTE_MAX_ETHPORTS;
2790         portid_t pl[RTE_MAX_ETHPORTS];
2791         portid_t peer_pl[RTE_MAX_ETHPORTS];
2792         uint16_t cnt_pi = 0;
2793         uint16_t cfg_pi = 0;
2794         int peer_pi;
2795         queueid_t qi;
2796         struct rte_port *port;
2797         struct rte_eth_hairpin_cap cap;
2798
2799         if (port_id_is_invalid(pid, ENABLED_WARN))
2800                 return 0;
2801
2802         RTE_ETH_FOREACH_DEV(pi) {
2803                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2804                         continue;
2805
2806                 if (port_is_bonding_slave(pi)) {
2807                         fprintf(stderr,
2808                                 "Please remove port %d from bonded device.\n",
2809                                 pi);
2810                         continue;
2811                 }
2812
2813                 need_check_link_status = 0;
2814                 port = &ports[pi];
2815                 if (port->port_status == RTE_PORT_STOPPED)
2816                         port->port_status = RTE_PORT_HANDLING;
2817                 else {
2818                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2819                         continue;
2820                 }
2821
2822                 if (port->need_reconfig > 0) {
2823                         struct rte_eth_conf dev_conf;
2824                         int k;
2825
2826                         port->need_reconfig = 0;
2827
2828                         if (flow_isolate_all) {
2829                                 int ret = port_flow_isolate(pi, 1);
2830                                 if (ret) {
2831                                         fprintf(stderr,
2832                                                 "Failed to apply isolated mode on port %d\n",
2833                                                 pi);
2834                                         return -1;
2835                                 }
2836                         }
2837                         configure_rxtx_dump_callbacks(0);
2838                         printf("Configuring Port %d (socket %u)\n", pi,
2839                                         port->socket_id);
2840                         if (nb_hairpinq > 0 &&
2841                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2842                                 fprintf(stderr,
2843                                         "Port %d doesn't support hairpin queues\n",
2844                                         pi);
2845                                 return -1;
2846                         }
2847
2848                         /* configure port */
2849                         diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2850                                                      nb_txq + nb_hairpinq,
2851                                                      &(port->dev_conf));
2852                         if (diag != 0) {
2853                                 if (port->port_status == RTE_PORT_HANDLING)
2854                                         port->port_status = RTE_PORT_STOPPED;
2855                                 else
2856                                         fprintf(stderr,
2857                                                 "Port %d can not be set back to stopped\n",
2858                                                 pi);
2859                                 fprintf(stderr, "Fail to configure port %d\n",
2860                                         pi);
2861                                 /* try to reconfigure port next time */
2862                                 port->need_reconfig = 1;
2863                                 return -1;
2864                         }
2865                         /* get device configuration*/
2866                         if (0 !=
2867                                 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2868                                 fprintf(stderr,
2869                                         "port %d can not get device configuration\n",
2870                                         pi);
2871                                 return -1;
2872                         }
2873                         /* Apply Rx offloads configuration */
2874                         if (dev_conf.rxmode.offloads !=
2875                             port->dev_conf.rxmode.offloads) {
2876                                 port->dev_conf.rxmode.offloads |=
2877                                         dev_conf.rxmode.offloads;
2878                                 for (k = 0;
2879                                      k < port->dev_info.max_rx_queues;
2880                                      k++)
2881                                         port->rx_conf[k].offloads |=
2882                                                 dev_conf.rxmode.offloads;
2883                         }
2884                         /* Apply Tx offloads configuration */
2885                         if (dev_conf.txmode.offloads !=
2886                             port->dev_conf.txmode.offloads) {
2887                                 port->dev_conf.txmode.offloads |=
2888                                         dev_conf.txmode.offloads;
2889                                 for (k = 0;
2890                                      k < port->dev_info.max_tx_queues;
2891                                      k++)
2892                                         port->tx_conf[k].offloads |=
2893                                                 dev_conf.txmode.offloads;
2894                         }
2895                 }
2896                 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2897                         port->need_reconfig_queues = 0;
2898                         /* setup tx queues */
2899                         for (qi = 0; qi < nb_txq; qi++) {
2900                                 if ((numa_support) &&
2901                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2902                                         diag = rte_eth_tx_queue_setup(pi, qi,
2903                                                 port->nb_tx_desc[qi],
2904                                                 txring_numa[pi],
2905                                                 &(port->tx_conf[qi]));
2906                                 else
2907                                         diag = rte_eth_tx_queue_setup(pi, qi,
2908                                                 port->nb_tx_desc[qi],
2909                                                 port->socket_id,
2910                                                 &(port->tx_conf[qi]));
2911
2912                                 if (diag == 0)
2913                                         continue;
2914
2915                                 /* Fail to setup tx queue, return */
2916                                 if (port->port_status == RTE_PORT_HANDLING)
2917                                         port->port_status = RTE_PORT_STOPPED;
2918                                 else
2919                                         fprintf(stderr,
2920                                                 "Port %d can not be set back to stopped\n",
2921                                                 pi);
2922                                 fprintf(stderr,
2923                                         "Fail to configure port %d tx queues\n",
2924                                         pi);
2925                                 /* try to reconfigure queues next time */
2926                                 port->need_reconfig_queues = 1;
2927                                 return -1;
2928                         }
2929                         for (qi = 0; qi < nb_rxq; qi++) {
2930                                 /* setup rx queues */
2931                                 if ((numa_support) &&
2932                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2933                                         struct rte_mempool * mp =
2934                                                 mbuf_pool_find
2935                                                         (rxring_numa[pi], 0);
2936                                         if (mp == NULL) {
2937                                                 fprintf(stderr,
2938                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2939                                                         rxring_numa[pi]);
2940                                                 return -1;
2941                                         }
2942
2943                                         diag = rx_queue_setup(pi, qi,
2944                                              port->nb_rx_desc[qi],
2945                                              rxring_numa[pi],
2946                                              &(port->rx_conf[qi]),
2947                                              mp);
2948                                 } else {
2949                                         struct rte_mempool *mp =
2950                                                 mbuf_pool_find
2951                                                         (port->socket_id, 0);
2952                                         if (mp == NULL) {
2953                                                 fprintf(stderr,
2954                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2955                                                         port->socket_id);
2956                                                 return -1;
2957                                         }
2958                                         diag = rx_queue_setup(pi, qi,
2959                                              port->nb_rx_desc[qi],
2960                                              port->socket_id,
2961                                              &(port->rx_conf[qi]),
2962                                              mp);
2963                                 }
2964                                 if (diag == 0)
2965                                         continue;
2966
2967                                 /* Fail to setup rx queue, return */
2968                                 if (port->port_status == RTE_PORT_HANDLING)
2969                                         port->port_status = RTE_PORT_STOPPED;
2970                                 else
2971                                         fprintf(stderr,
2972                                                 "Port %d can not be set back to stopped\n",
2973                                                 pi);
2974                                 fprintf(stderr,
2975                                         "Fail to configure port %d rx queues\n",
2976                                         pi);
2977                                 /* try to reconfigure queues next time */
2978                                 port->need_reconfig_queues = 1;
2979                                 return -1;
2980                         }
2981                         /* setup hairpin queues */
2982                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2983                                 return -1;
2984                 }
2985                 configure_rxtx_dump_callbacks(verbose_level);
2986                 if (clear_ptypes) {
2987                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2988                                         NULL, 0);
2989                         if (diag < 0)
2990                                 fprintf(stderr,
2991                                         "Port %d: Failed to disable Ptype parsing\n",
2992                                         pi);
2993                 }
2994
2995                 p_pi = pi;
2996                 cnt_pi++;
2997
2998                 /* start port */
2999                 diag = eth_dev_start_mp(pi);
3000                 if (diag < 0) {
3001                         fprintf(stderr, "Fail to start port %d: %s\n",
3002                                 pi, rte_strerror(-diag));
3003
3004                         /* Fail to setup rx queue, return */
3005                         if (port->port_status == RTE_PORT_HANDLING)
3006                                 port->port_status = RTE_PORT_STOPPED;
3007                         else
3008                                 fprintf(stderr,
3009                                         "Port %d can not be set back to stopped\n",
3010                                         pi);
3011                         continue;
3012                 }
3013
3014                 if (port->port_status == RTE_PORT_HANDLING)
3015                         port->port_status = RTE_PORT_STARTED;
3016                 else
3017                         fprintf(stderr, "Port %d can not be set into started\n",
3018                                 pi);
3019
3020                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
3021                         printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
3022                                         RTE_ETHER_ADDR_BYTES(&port->eth_addr));
3023
3024                 /* at least one port started, need checking link status */
3025                 need_check_link_status = 1;
3026
3027                 pl[cfg_pi++] = pi;
3028         }
3029
3030         if (need_check_link_status == 1 && !no_link_check)
3031                 check_all_ports_link_status(RTE_PORT_ALL);
3032         else if (need_check_link_status == 0)
3033                 fprintf(stderr, "Please stop the ports first\n");
3034
3035         if (hairpin_mode & 0xf) {
3036                 uint16_t i;
3037                 int j;
3038
3039                 /* bind all started hairpin ports */
3040                 for (i = 0; i < cfg_pi; i++) {
3041                         pi = pl[i];
3042                         /* bind current Tx to all peer Rx */
3043                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3044                                                         RTE_MAX_ETHPORTS, 1);
3045                         if (peer_pi < 0)
3046                                 return peer_pi;
3047                         for (j = 0; j < peer_pi; j++) {
3048                                 if (!port_is_started(peer_pl[j]))
3049                                         continue;
3050                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
3051                                 if (diag < 0) {
3052                                         fprintf(stderr,
3053                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3054                                                 pi, peer_pl[j],
3055                                                 rte_strerror(-diag));
3056                                         return -1;
3057                                 }
3058                         }
3059                         /* bind all peer Tx to current Rx */
3060                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3061                                                         RTE_MAX_ETHPORTS, 0);
3062                         if (peer_pi < 0)
3063                                 return peer_pi;
3064                         for (j = 0; j < peer_pi; j++) {
3065                                 if (!port_is_started(peer_pl[j]))
3066                                         continue;
3067                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
3068                                 if (diag < 0) {
3069                                         fprintf(stderr,
3070                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3071                                                 peer_pl[j], pi,
3072                                                 rte_strerror(-diag));
3073                                         return -1;
3074                                 }
3075                         }
3076                 }
3077         }
3078
3079         fill_xstats_display_info_for_port(pid);
3080
3081         printf("Done\n");
3082         return 0;
3083 }
3084
3085 void
3086 stop_port(portid_t pid)
3087 {
3088         portid_t pi;
3089         struct rte_port *port;
3090         int need_check_link_status = 0;
3091         portid_t peer_pl[RTE_MAX_ETHPORTS];
3092         int peer_pi;
3093
3094         if (port_id_is_invalid(pid, ENABLED_WARN))
3095                 return;
3096
3097         printf("Stopping ports...\n");
3098
3099         RTE_ETH_FOREACH_DEV(pi) {
3100                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3101                         continue;
3102
3103                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3104                         fprintf(stderr,
3105                                 "Please remove port %d from forwarding configuration.\n",
3106                                 pi);
3107                         continue;
3108                 }
3109
3110                 if (port_is_bonding_slave(pi)) {
3111                         fprintf(stderr,
3112                                 "Please remove port %d from bonded device.\n",
3113                                 pi);
3114                         continue;
3115                 }
3116
3117                 port = &ports[pi];
3118                 if (port->port_status == RTE_PORT_STARTED)
3119                         port->port_status = RTE_PORT_HANDLING;
3120                 else
3121                         continue;
3122
3123                 if (hairpin_mode & 0xf) {
3124                         int j;
3125
3126                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3127                         /* unbind all peer Tx from current Rx */
3128                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3129                                                         RTE_MAX_ETHPORTS, 0);
3130                         if (peer_pi < 0)
3131                                 continue;
3132                         for (j = 0; j < peer_pi; j++) {
3133                                 if (!port_is_started(peer_pl[j]))
3134                                         continue;
3135                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
3136                         }
3137                 }
3138
3139                 if (port->flow_list)
3140                         port_flow_flush(pi);
3141
3142                 if (eth_dev_stop_mp(pi) != 0)
3143                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3144                                 pi);
3145
3146                 if (port->port_status == RTE_PORT_HANDLING)
3147                         port->port_status = RTE_PORT_STOPPED;
3148                 else
3149                         fprintf(stderr, "Port %d can not be set into stopped\n",
3150                                 pi);
3151                 need_check_link_status = 1;
3152         }
3153         if (need_check_link_status && !no_link_check)
3154                 check_all_ports_link_status(RTE_PORT_ALL);
3155
3156         printf("Done\n");
3157 }
3158
3159 static void
3160 remove_invalid_ports_in(portid_t *array, portid_t *total)
3161 {
3162         portid_t i;
3163         portid_t new_total = 0;
3164
3165         for (i = 0; i < *total; i++)
3166                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3167                         array[new_total] = array[i];
3168                         new_total++;
3169                 }
3170         *total = new_total;
3171 }
3172
3173 static void
3174 remove_invalid_ports(void)
3175 {
3176         remove_invalid_ports_in(ports_ids, &nb_ports);
3177         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3178         nb_cfg_ports = nb_fwd_ports;
3179 }
3180
3181 void
3182 close_port(portid_t pid)
3183 {
3184         portid_t pi;
3185         struct rte_port *port;
3186
3187         if (port_id_is_invalid(pid, ENABLED_WARN))
3188                 return;
3189
3190         printf("Closing ports...\n");
3191
3192         RTE_ETH_FOREACH_DEV(pi) {
3193                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3194                         continue;
3195
3196                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3197                         fprintf(stderr,
3198                                 "Please remove port %d from forwarding configuration.\n",
3199                                 pi);
3200                         continue;
3201                 }
3202
3203                 if (port_is_bonding_slave(pi)) {
3204                         fprintf(stderr,
3205                                 "Please remove port %d from bonded device.\n",
3206                                 pi);
3207                         continue;
3208                 }
3209
3210                 port = &ports[pi];
3211                 if (port->port_status == RTE_PORT_CLOSED) {
3212                         fprintf(stderr, "Port %d is already closed\n", pi);
3213                         continue;
3214                 }
3215
3216                 if (is_proc_primary()) {
3217                         port_flow_flush(pi);
3218                         port_flex_item_flush(pi);
3219                         rte_eth_dev_close(pi);
3220                 }
3221
3222                 free_xstats_display_info(pi);
3223         }
3224
3225         remove_invalid_ports();
3226         printf("Done\n");
3227 }
3228
3229 void
3230 reset_port(portid_t pid)
3231 {
3232         int diag;
3233         portid_t pi;
3234         struct rte_port *port;
3235
3236         if (port_id_is_invalid(pid, ENABLED_WARN))
3237                 return;
3238
3239         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3240                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3241                 fprintf(stderr,
3242                         "Can not reset port(s), please stop port(s) first.\n");
3243                 return;
3244         }
3245
3246         printf("Resetting ports...\n");
3247
3248         RTE_ETH_FOREACH_DEV(pi) {
3249                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3250                         continue;
3251
3252                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3253                         fprintf(stderr,
3254                                 "Please remove port %d from forwarding configuration.\n",
3255                                 pi);
3256                         continue;
3257                 }
3258
3259                 if (port_is_bonding_slave(pi)) {
3260                         fprintf(stderr,
3261                                 "Please remove port %d from bonded device.\n",
3262                                 pi);
3263                         continue;
3264                 }
3265
3266                 diag = rte_eth_dev_reset(pi);
3267                 if (diag == 0) {
3268                         port = &ports[pi];
3269                         port->need_reconfig = 1;
3270                         port->need_reconfig_queues = 1;
3271                 } else {
3272                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3273                                 pi, diag);
3274                 }
3275         }
3276
3277         printf("Done\n");
3278 }
3279
3280 void
3281 attach_port(char *identifier)
3282 {
3283         portid_t pi;
3284         struct rte_dev_iterator iterator;
3285
3286         printf("Attaching a new port...\n");
3287
3288         if (identifier == NULL) {
3289                 fprintf(stderr, "Invalid parameters are specified\n");
3290                 return;
3291         }
3292
3293         if (rte_dev_probe(identifier) < 0) {
3294                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3295                 return;
3296         }
3297
3298         /* first attach mode: event */
3299         if (setup_on_probe_event) {
3300                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3301                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3302                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
3303                                         ports[pi].need_setup != 0)
3304                                 setup_attached_port(pi);
3305                 return;
3306         }
3307
3308         /* second attach mode: iterator */
3309         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3310                 /* setup ports matching the devargs used for probing */
3311                 if (port_is_forwarding(pi))
3312                         continue; /* port was already attached before */
3313                 setup_attached_port(pi);
3314         }
3315 }
3316
3317 static void
3318 setup_attached_port(portid_t pi)
3319 {
3320         unsigned int socket_id;
3321         int ret;
3322
3323         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3324         /* if socket_id is invalid, set to the first available socket. */
3325         if (check_socket_id(socket_id) < 0)
3326                 socket_id = socket_ids[0];
3327         reconfig(pi, socket_id);
3328         ret = rte_eth_promiscuous_enable(pi);
3329         if (ret != 0)
3330                 fprintf(stderr,
3331                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3332                         pi, rte_strerror(-ret));
3333
3334         ports_ids[nb_ports++] = pi;
3335         fwd_ports_ids[nb_fwd_ports++] = pi;
3336         nb_cfg_ports = nb_fwd_ports;
3337         ports[pi].need_setup = 0;
3338         ports[pi].port_status = RTE_PORT_STOPPED;
3339
3340         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3341         printf("Done\n");
3342 }
3343
3344 static void
3345 detach_device(struct rte_device *dev)
3346 {
3347         portid_t sibling;
3348
3349         if (dev == NULL) {
3350                 fprintf(stderr, "Device already removed\n");
3351                 return;
3352         }
3353
3354         printf("Removing a device...\n");
3355
3356         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3357                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3358                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3359                                 fprintf(stderr, "Port %u not stopped\n",
3360                                         sibling);
3361                                 return;
3362                         }
3363                         port_flow_flush(sibling);
3364                 }
3365         }
3366
3367         if (rte_dev_remove(dev) < 0) {
3368                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3369                 return;
3370         }
3371         remove_invalid_ports();
3372
3373         printf("Device is detached\n");
3374         printf("Now total ports is %d\n", nb_ports);
3375         printf("Done\n");
3376         return;
3377 }
3378
3379 void
3380 detach_port_device(portid_t port_id)
3381 {
3382         int ret;
3383         struct rte_eth_dev_info dev_info;
3384
3385         if (port_id_is_invalid(port_id, ENABLED_WARN))
3386                 return;
3387
3388         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3389                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3390                         fprintf(stderr, "Port not stopped\n");
3391                         return;
3392                 }
3393                 fprintf(stderr, "Port was not closed\n");
3394         }
3395
3396         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3397         if (ret != 0) {
3398                 TESTPMD_LOG(ERR,
3399                         "Failed to get device info for port %d, not detaching\n",
3400                         port_id);
3401                 return;
3402         }
3403         detach_device(dev_info.device);
3404 }
3405
3406 void
3407 detach_devargs(char *identifier)
3408 {
3409         struct rte_dev_iterator iterator;
3410         struct rte_devargs da;
3411         portid_t port_id;
3412
3413         printf("Removing a device...\n");
3414
3415         memset(&da, 0, sizeof(da));
3416         if (rte_devargs_parsef(&da, "%s", identifier)) {
3417                 fprintf(stderr, "cannot parse identifier\n");
3418                 return;
3419         }
3420
3421         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3422                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3423                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3424                                 fprintf(stderr, "Port %u not stopped\n",
3425                                         port_id);
3426                                 rte_eth_iterator_cleanup(&iterator);
3427                                 rte_devargs_reset(&da);
3428                                 return;
3429                         }
3430                         port_flow_flush(port_id);
3431                 }
3432         }
3433
3434         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3435                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3436                             da.name, da.bus->name);
3437                 rte_devargs_reset(&da);
3438                 return;
3439         }
3440
3441         remove_invalid_ports();
3442
3443         printf("Device %s is detached\n", identifier);
3444         printf("Now total ports is %d\n", nb_ports);
3445         printf("Done\n");
3446         rte_devargs_reset(&da);
3447 }
3448
3449 void
3450 pmd_test_exit(void)
3451 {
3452         portid_t pt_id;
3453         unsigned int i;
3454         int ret;
3455
3456         if (test_done == 0)
3457                 stop_packet_forwarding();
3458
3459 #ifndef RTE_EXEC_ENV_WINDOWS
3460         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3461                 if (mempools[i]) {
3462                         if (mp_alloc_type == MP_ALLOC_ANON)
3463                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3464                                                      NULL);
3465                 }
3466         }
3467 #endif
3468         if (ports != NULL) {
3469                 no_link_check = 1;
3470                 RTE_ETH_FOREACH_DEV(pt_id) {
3471                         printf("\nStopping port %d...\n", pt_id);
3472                         fflush(stdout);
3473                         stop_port(pt_id);
3474                 }
3475                 RTE_ETH_FOREACH_DEV(pt_id) {
3476                         printf("\nShutting down port %d...\n", pt_id);
3477                         fflush(stdout);
3478                         close_port(pt_id);
3479                 }
3480         }
3481
3482         if (hot_plug) {
3483                 ret = rte_dev_event_monitor_stop();
3484                 if (ret) {
3485                         RTE_LOG(ERR, EAL,
3486                                 "fail to stop device event monitor.");
3487                         return;
3488                 }
3489
3490                 ret = rte_dev_event_callback_unregister(NULL,
3491                         dev_event_callback, NULL);
3492                 if (ret < 0) {
3493                         RTE_LOG(ERR, EAL,
3494                                 "fail to unregister device event callback.\n");
3495                         return;
3496                 }
3497
3498                 ret = rte_dev_hotplug_handle_disable();
3499                 if (ret) {
3500                         RTE_LOG(ERR, EAL,
3501                                 "fail to disable hotplug handling.\n");
3502                         return;
3503                 }
3504         }
3505         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3506                 if (mempools[i])
3507                         mempool_free_mp(mempools[i]);
3508         }
3509         free(xstats_display);
3510
3511         printf("\nBye...\n");
3512 }
3513
3514 typedef void (*cmd_func_t)(void);
3515 struct pmd_test_command {
3516         const char *cmd_name;
3517         cmd_func_t cmd_func;
3518 };
3519
3520 /* Check the link status of all ports in up to 9s, and print them finally */
3521 static void
3522 check_all_ports_link_status(uint32_t port_mask)
3523 {
3524 #define CHECK_INTERVAL 100 /* 100ms */
3525 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3526         portid_t portid;
3527         uint8_t count, all_ports_up, print_flag = 0;
3528         struct rte_eth_link link;
3529         int ret;
3530         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3531
3532         printf("Checking link statuses...\n");
3533         fflush(stdout);
3534         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3535                 all_ports_up = 1;
3536                 RTE_ETH_FOREACH_DEV(portid) {
3537                         if ((port_mask & (1 << portid)) == 0)
3538                                 continue;
3539                         memset(&link, 0, sizeof(link));
3540                         ret = rte_eth_link_get_nowait(portid, &link);
3541                         if (ret < 0) {
3542                                 all_ports_up = 0;
3543                                 if (print_flag == 1)
3544                                         fprintf(stderr,
3545                                                 "Port %u link get failed: %s\n",
3546                                                 portid, rte_strerror(-ret));
3547                                 continue;
3548                         }
3549                         /* print link status if flag set */
3550                         if (print_flag == 1) {
3551                                 rte_eth_link_to_str(link_status,
3552                                         sizeof(link_status), &link);
3553                                 printf("Port %d %s\n", portid, link_status);
3554                                 continue;
3555                         }
3556                         /* clear all_ports_up flag if any link down */
3557                         if (link.link_status == RTE_ETH_LINK_DOWN) {
3558                                 all_ports_up = 0;
3559                                 break;
3560                         }
3561                 }
3562                 /* after finally printing all link status, get out */
3563                 if (print_flag == 1)
3564                         break;
3565
3566                 if (all_ports_up == 0) {
3567                         fflush(stdout);
3568                         rte_delay_ms(CHECK_INTERVAL);
3569                 }
3570
3571                 /* set the print_flag if all ports up or timeout */
3572                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3573                         print_flag = 1;
3574                 }
3575
3576                 if (lsc_interrupt)
3577                         break;
3578         }
3579 }
3580
3581 static void
3582 rmv_port_callback(void *arg)
3583 {
3584         int need_to_start = 0;
3585         int org_no_link_check = no_link_check;
3586         portid_t port_id = (intptr_t)arg;
3587         struct rte_eth_dev_info dev_info;
3588         int ret;
3589
3590         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3591
3592         if (!test_done && port_is_forwarding(port_id)) {
3593                 need_to_start = 1;
3594                 stop_packet_forwarding();
3595         }
3596         no_link_check = 1;
3597         stop_port(port_id);
3598         no_link_check = org_no_link_check;
3599
3600         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3601         if (ret != 0)
3602                 TESTPMD_LOG(ERR,
3603                         "Failed to get device info for port %d, not detaching\n",
3604                         port_id);
3605         else {
3606                 struct rte_device *device = dev_info.device;
3607                 close_port(port_id);
3608                 detach_device(device); /* might be already removed or have more ports */
3609         }
3610         if (need_to_start)
3611                 start_packet_forwarding(0);
3612 }
3613
3614 /* This function is used by the interrupt thread */
3615 static int
3616 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3617                   void *ret_param)
3618 {
3619         RTE_SET_USED(param);
3620         RTE_SET_USED(ret_param);
3621
3622         if (type >= RTE_ETH_EVENT_MAX) {
3623                 fprintf(stderr,
3624                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3625                         port_id, __func__, type);
3626                 fflush(stderr);
3627         } else if (event_print_mask & (UINT32_C(1) << type)) {
3628                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3629                         eth_event_desc[type]);
3630                 fflush(stdout);
3631         }
3632
3633         switch (type) {
3634         case RTE_ETH_EVENT_NEW:
3635                 ports[port_id].need_setup = 1;
3636                 ports[port_id].port_status = RTE_PORT_HANDLING;
3637                 break;
3638         case RTE_ETH_EVENT_INTR_RMV:
3639                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3640                         break;
3641                 if (rte_eal_alarm_set(100000,
3642                                 rmv_port_callback, (void *)(intptr_t)port_id))
3643                         fprintf(stderr,
3644                                 "Could not set up deferred device removal\n");
3645                 break;
3646         case RTE_ETH_EVENT_DESTROY:
3647                 ports[port_id].port_status = RTE_PORT_CLOSED;
3648                 printf("Port %u is closed\n", port_id);
3649                 break;
3650         default:
3651                 break;
3652         }
3653         return 0;
3654 }
3655
3656 static int
3657 register_eth_event_callback(void)
3658 {
3659         int ret;
3660         enum rte_eth_event_type event;
3661
3662         for (event = RTE_ETH_EVENT_UNKNOWN;
3663                         event < RTE_ETH_EVENT_MAX; event++) {
3664                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3665                                 event,
3666                                 eth_event_callback,
3667                                 NULL);
3668                 if (ret != 0) {
3669                         TESTPMD_LOG(ERR, "Failed to register callback for "
3670                                         "%s event\n", eth_event_desc[event]);
3671                         return -1;
3672                 }
3673         }
3674
3675         return 0;
3676 }
3677
3678 /* This function is used by the interrupt thread */
3679 static void
3680 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3681                              __rte_unused void *arg)
3682 {
3683         uint16_t port_id;
3684         int ret;
3685
3686         if (type >= RTE_DEV_EVENT_MAX) {
3687                 fprintf(stderr, "%s called upon invalid event %d\n",
3688                         __func__, type);
3689                 fflush(stderr);
3690         }
3691
3692         switch (type) {
3693         case RTE_DEV_EVENT_REMOVE:
3694                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3695                         device_name);
3696                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3697                 if (ret) {
3698                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3699                                 device_name);
3700                         return;
3701                 }
3702                 /*
3703                  * Because the user's callback is invoked in eal interrupt
3704                  * callback, the interrupt callback need to be finished before
3705                  * it can be unregistered when detaching device. So finish
3706                  * callback soon and use a deferred removal to detach device
3707                  * is need. It is a workaround, once the device detaching be
3708                  * moved into the eal in the future, the deferred removal could
3709                  * be deleted.
3710                  */
3711                 if (rte_eal_alarm_set(100000,
3712                                 rmv_port_callback, (void *)(intptr_t)port_id))
3713                         RTE_LOG(ERR, EAL,
3714                                 "Could not set up deferred device removal\n");
3715                 break;
3716         case RTE_DEV_EVENT_ADD:
3717                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3718                         device_name);
3719                 /* TODO: After finish kernel driver binding,
3720                  * begin to attach port.
3721                  */
3722                 break;
3723         default:
3724                 break;
3725         }
3726 }
3727
3728 static void
3729 rxtx_port_config(portid_t pid)
3730 {
3731         uint16_t qid;
3732         uint64_t offloads;
3733         struct rte_port *port = &ports[pid];
3734
3735         for (qid = 0; qid < nb_rxq; qid++) {
3736                 offloads = port->rx_conf[qid].offloads;
3737                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3738
3739                 if (rxq_share > 0 &&
3740                     (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3741                         /* Non-zero share group to enable RxQ share. */
3742                         port->rx_conf[qid].share_group = pid / rxq_share + 1;
3743                         port->rx_conf[qid].share_qid = qid; /* Equal mapping. */
3744                 }
3745
3746                 if (offloads != 0)
3747                         port->rx_conf[qid].offloads = offloads;
3748
3749                 /* Check if any Rx parameters have been passed */
3750                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3751                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3752
3753                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3754                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3755
3756                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3757                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3758
3759                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3760                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3761
3762                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3763                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3764
3765                 port->nb_rx_desc[qid] = nb_rxd;
3766         }
3767
3768         for (qid = 0; qid < nb_txq; qid++) {
3769                 offloads = port->tx_conf[qid].offloads;
3770                 port->tx_conf[qid] = port->dev_info.default_txconf;
3771                 if (offloads != 0)
3772                         port->tx_conf[qid].offloads = offloads;
3773
3774                 /* Check if any Tx parameters have been passed */
3775                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3776                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3777
3778                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3779                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3780
3781                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3782                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3783
3784                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3785                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3786
3787                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3788                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3789
3790                 port->nb_tx_desc[qid] = nb_txd;
3791         }
3792 }
3793
3794 /*
3795  * Helper function to set MTU from frame size
3796  *
3797  * port->dev_info should be set before calling this function.
3798  *
3799  * return 0 on success, negative on error
3800  */
3801 int
3802 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3803 {
3804         struct rte_port *port = &ports[portid];
3805         uint32_t eth_overhead;
3806         uint16_t mtu, new_mtu;
3807
3808         eth_overhead = get_eth_overhead(&port->dev_info);
3809
3810         if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3811                 printf("Failed to get MTU for port %u\n", portid);
3812                 return -1;
3813         }
3814
3815         new_mtu = max_rx_pktlen - eth_overhead;
3816
3817         if (mtu == new_mtu)
3818                 return 0;
3819
3820         if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3821                 fprintf(stderr,
3822                         "Failed to set MTU to %u for port %u\n",
3823                         new_mtu, portid);
3824                 return -1;
3825         }
3826
3827         port->dev_conf.rxmode.mtu = new_mtu;
3828
3829         return 0;
3830 }
3831
3832 void
3833 init_port_config(void)
3834 {
3835         portid_t pid;
3836         struct rte_port *port;
3837         int ret, i;
3838
3839         RTE_ETH_FOREACH_DEV(pid) {
3840                 port = &ports[pid];
3841                 port->dev_conf.fdir_conf = fdir_conf;
3842
3843                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3844                 if (ret != 0)
3845                         return;
3846
3847                 if (nb_rxq > 1) {
3848                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3849                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3850                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3851                 } else {
3852                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3853                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3854                 }
3855
3856                 if (port->dcb_flag == 0) {
3857                         if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3858                                 port->dev_conf.rxmode.mq_mode =
3859                                         (enum rte_eth_rx_mq_mode)
3860                                                 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3861                         } else {
3862                                 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3863                                 port->dev_conf.rxmode.offloads &=
3864                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3865
3866                                 for (i = 0;
3867                                      i < port->dev_info.nb_rx_queues;
3868                                      i++)
3869                                         port->rx_conf[i].offloads &=
3870                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3871                         }
3872                 }
3873
3874                 rxtx_port_config(pid);
3875
3876                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3877                 if (ret != 0)
3878                         return;
3879
3880 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3881                 rte_pmd_ixgbe_bypass_init(pid);
3882 #endif
3883
3884                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3885                         port->dev_conf.intr_conf.lsc = 1;
3886                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3887                         port->dev_conf.intr_conf.rmv = 1;
3888         }
3889 }
3890
3891 void set_port_slave_flag(portid_t slave_pid)
3892 {
3893         struct rte_port *port;
3894
3895         port = &ports[slave_pid];
3896         port->slave_flag = 1;
3897 }
3898
3899 void clear_port_slave_flag(portid_t slave_pid)
3900 {
3901         struct rte_port *port;
3902
3903         port = &ports[slave_pid];
3904         port->slave_flag = 0;
3905 }
3906
3907 uint8_t port_is_bonding_slave(portid_t slave_pid)
3908 {
3909         struct rte_port *port;
3910         struct rte_eth_dev_info dev_info;
3911         int ret;
3912
3913         port = &ports[slave_pid];
3914         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3915         if (ret != 0) {
3916                 TESTPMD_LOG(ERR,
3917                         "Failed to get device info for port id %d,"
3918                         "cannot determine if the port is a bonded slave",
3919                         slave_pid);
3920                 return 0;
3921         }
3922         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3923                 return 1;
3924         return 0;
3925 }
3926
3927 const uint16_t vlan_tags[] = {
3928                 0,  1,  2,  3,  4,  5,  6,  7,
3929                 8,  9, 10, 11,  12, 13, 14, 15,
3930                 16, 17, 18, 19, 20, 21, 22, 23,
3931                 24, 25, 26, 27, 28, 29, 30, 31
3932 };
3933
3934 static  int
3935 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3936                  enum dcb_mode_enable dcb_mode,
3937                  enum rte_eth_nb_tcs num_tcs,
3938                  uint8_t pfc_en)
3939 {
3940         uint8_t i;
3941         int32_t rc;
3942         struct rte_eth_rss_conf rss_conf;
3943
3944         /*
3945          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3946          * given above, and the number of traffic classes available for use.
3947          */
3948         if (dcb_mode == DCB_VT_ENABLED) {
3949                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3950                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3951                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3952                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3953
3954                 /* VMDQ+DCB RX and TX configurations */
3955                 vmdq_rx_conf->enable_default_pool = 0;
3956                 vmdq_rx_conf->default_pool = 0;
3957                 vmdq_rx_conf->nb_queue_pools =
3958                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3959                 vmdq_tx_conf->nb_queue_pools =
3960                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3961
3962                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3963                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3964                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3965                         vmdq_rx_conf->pool_map[i].pools =
3966                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3967                 }
3968                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3969                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3970                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3971                 }
3972
3973                 /* set DCB mode of RX and TX of multiple queues */
3974                 eth_conf->rxmode.mq_mode =
3975                                 (enum rte_eth_rx_mq_mode)
3976                                         (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
3977                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
3978         } else {
3979                 struct rte_eth_dcb_rx_conf *rx_conf =
3980                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3981                 struct rte_eth_dcb_tx_conf *tx_conf =
3982                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3983
3984                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3985
3986                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3987                 if (rc != 0)
3988                         return rc;
3989
3990                 rx_conf->nb_tcs = num_tcs;
3991                 tx_conf->nb_tcs = num_tcs;
3992
3993                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3994                         rx_conf->dcb_tc[i] = i % num_tcs;
3995                         tx_conf->dcb_tc[i] = i % num_tcs;
3996                 }
3997
3998                 eth_conf->rxmode.mq_mode =
3999                                 (enum rte_eth_rx_mq_mode)
4000                                         (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
4001                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
4002                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
4003         }
4004
4005         if (pfc_en)
4006                 eth_conf->dcb_capability_en =
4007                                 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
4008         else
4009                 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
4010
4011         return 0;
4012 }
4013
4014 int
4015 init_port_dcb_config(portid_t pid,
4016                      enum dcb_mode_enable dcb_mode,
4017                      enum rte_eth_nb_tcs num_tcs,
4018                      uint8_t pfc_en)
4019 {
4020         struct rte_eth_conf port_conf;
4021         struct rte_port *rte_port;
4022         int retval;
4023         uint16_t i;
4024
4025         if (num_procs > 1) {
4026                 printf("The multi-process feature doesn't support dcb.\n");
4027                 return -ENOTSUP;
4028         }
4029         rte_port = &ports[pid];
4030
4031         /* retain the original device configuration. */
4032         memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
4033
4034         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
4035         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
4036         if (retval < 0)
4037                 return retval;
4038         port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4039         /* remove RSS HASH offload for DCB in vt mode */
4040         if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
4041                 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4042                 for (i = 0; i < nb_rxq; i++)
4043                         rte_port->rx_conf[i].offloads &=
4044                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4045         }
4046
4047         /* re-configure the device . */
4048         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
4049         if (retval < 0)
4050                 return retval;
4051
4052         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
4053         if (retval != 0)
4054                 return retval;
4055
4056         /* If dev_info.vmdq_pool_base is greater than 0,
4057          * the queue id of vmdq pools is started after pf queues.
4058          */
4059         if (dcb_mode == DCB_VT_ENABLED &&
4060             rte_port->dev_info.vmdq_pool_base > 0) {
4061                 fprintf(stderr,
4062                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
4063                         pid);
4064                 return -1;
4065         }
4066
4067         /* Assume the ports in testpmd have the same dcb capability
4068          * and has the same number of rxq and txq in dcb mode
4069          */
4070         if (dcb_mode == DCB_VT_ENABLED) {
4071                 if (rte_port->dev_info.max_vfs > 0) {
4072                         nb_rxq = rte_port->dev_info.nb_rx_queues;
4073                         nb_txq = rte_port->dev_info.nb_tx_queues;
4074                 } else {
4075                         nb_rxq = rte_port->dev_info.max_rx_queues;
4076                         nb_txq = rte_port->dev_info.max_tx_queues;
4077                 }
4078         } else {
4079                 /*if vt is disabled, use all pf queues */
4080                 if (rte_port->dev_info.vmdq_pool_base == 0) {
4081                         nb_rxq = rte_port->dev_info.max_rx_queues;
4082                         nb_txq = rte_port->dev_info.max_tx_queues;
4083                 } else {
4084                         nb_rxq = (queueid_t)num_tcs;
4085                         nb_txq = (queueid_t)num_tcs;
4086
4087                 }
4088         }
4089         rx_free_thresh = 64;
4090
4091         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4092
4093         rxtx_port_config(pid);
4094         /* VLAN filter */
4095         rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4096         for (i = 0; i < RTE_DIM(vlan_tags); i++)
4097                 rx_vft_set(pid, vlan_tags[i], 1);
4098
4099         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4100         if (retval != 0)
4101                 return retval;
4102
4103         rte_port->dcb_flag = 1;
4104
4105         /* Enter DCB configuration status */
4106         dcb_config = 1;
4107
4108         return 0;
4109 }
4110
4111 static void
4112 init_port(void)
4113 {
4114         int i;
4115
4116         /* Configuration of Ethernet ports. */
4117         ports = rte_zmalloc("testpmd: ports",
4118                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4119                             RTE_CACHE_LINE_SIZE);
4120         if (ports == NULL) {
4121                 rte_exit(EXIT_FAILURE,
4122                                 "rte_zmalloc(%d struct rte_port) failed\n",
4123                                 RTE_MAX_ETHPORTS);
4124         }
4125         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4126                 ports[i].xstats_info.allocated = false;
4127         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4128                 LIST_INIT(&ports[i].flow_tunnel_list);
4129         /* Initialize ports NUMA structures */
4130         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4131         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4132         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4133 }
4134
4135 static void
4136 force_quit(void)
4137 {
4138         pmd_test_exit();
4139         prompt_exit();
4140 }
4141
4142 static void
4143 print_stats(void)
4144 {
4145         uint8_t i;
4146         const char clr[] = { 27, '[', '2', 'J', '\0' };
4147         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4148
4149         /* Clear screen and move to top left */
4150         printf("%s%s", clr, top_left);
4151
4152         printf("\nPort statistics ====================================");
4153         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4154                 nic_stats_display(fwd_ports_ids[i]);
4155
4156         fflush(stdout);
4157 }
4158
4159 static void
4160 signal_handler(int signum)
4161 {
4162         if (signum == SIGINT || signum == SIGTERM) {
4163                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4164                         signum);
4165 #ifdef RTE_LIB_PDUMP
4166                 /* uninitialize packet capture framework */
4167                 rte_pdump_uninit();
4168 #endif
4169 #ifdef RTE_LIB_LATENCYSTATS
4170                 if (latencystats_enabled != 0)
4171                         rte_latencystats_uninit();
4172 #endif
4173                 force_quit();
4174                 /* Set flag to indicate the force termination. */
4175                 f_quit = 1;
4176                 /* exit with the expected status */
4177 #ifndef RTE_EXEC_ENV_WINDOWS
4178                 signal(signum, SIG_DFL);
4179                 kill(getpid(), signum);
4180 #endif
4181         }
4182 }
4183
4184 int
4185 main(int argc, char** argv)
4186 {
4187         int diag;
4188         portid_t port_id;
4189         uint16_t count;
4190         int ret;
4191
4192         signal(SIGINT, signal_handler);
4193         signal(SIGTERM, signal_handler);
4194
4195         testpmd_logtype = rte_log_register("testpmd");
4196         if (testpmd_logtype < 0)
4197                 rte_exit(EXIT_FAILURE, "Cannot register log type");
4198         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4199
4200         diag = rte_eal_init(argc, argv);
4201         if (diag < 0)
4202                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4203                          rte_strerror(rte_errno));
4204
4205         ret = register_eth_event_callback();
4206         if (ret != 0)
4207                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4208
4209 #ifdef RTE_LIB_PDUMP
4210         /* initialize packet capture framework */
4211         rte_pdump_init();
4212 #endif
4213
4214         count = 0;
4215         RTE_ETH_FOREACH_DEV(port_id) {
4216                 ports_ids[count] = port_id;
4217                 count++;
4218         }
4219         nb_ports = (portid_t) count;
4220         if (nb_ports == 0)
4221                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4222
4223         /* allocate port structures, and init them */
4224         init_port();
4225
4226         set_def_fwd_config();
4227         if (nb_lcores == 0)
4228                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4229                          "Check the core mask argument\n");
4230
4231         /* Bitrate/latency stats disabled by default */
4232 #ifdef RTE_LIB_BITRATESTATS
4233         bitrate_enabled = 0;
4234 #endif
4235 #ifdef RTE_LIB_LATENCYSTATS
4236         latencystats_enabled = 0;
4237 #endif
4238
4239         /* on FreeBSD, mlockall() is disabled by default */
4240 #ifdef RTE_EXEC_ENV_FREEBSD
4241         do_mlockall = 0;
4242 #else
4243         do_mlockall = 1;
4244 #endif
4245
4246         argc -= diag;
4247         argv += diag;
4248         if (argc > 1)
4249                 launch_args_parse(argc, argv);
4250
4251 #ifndef RTE_EXEC_ENV_WINDOWS
4252         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4253                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4254                         strerror(errno));
4255         }
4256 #endif
4257
4258         if (tx_first && interactive)
4259                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4260                                 "interactive mode.\n");
4261
4262         if (tx_first && lsc_interrupt) {
4263                 fprintf(stderr,
4264                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4265                 lsc_interrupt = 0;
4266         }
4267
4268         if (!nb_rxq && !nb_txq)
4269                 fprintf(stderr,
4270                         "Warning: Either rx or tx queues should be non-zero\n");
4271
4272         if (nb_rxq > 1 && nb_rxq > nb_txq)
4273                 fprintf(stderr,
4274                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4275                         nb_rxq, nb_txq);
4276
4277         init_config();
4278
4279         if (hot_plug) {
4280                 ret = rte_dev_hotplug_handle_enable();
4281                 if (ret) {
4282                         RTE_LOG(ERR, EAL,
4283                                 "fail to enable hotplug handling.");
4284                         return -1;
4285                 }
4286
4287                 ret = rte_dev_event_monitor_start();
4288                 if (ret) {
4289                         RTE_LOG(ERR, EAL,
4290                                 "fail to start device event monitoring.");
4291                         return -1;
4292                 }
4293
4294                 ret = rte_dev_event_callback_register(NULL,
4295                         dev_event_callback, NULL);
4296                 if (ret) {
4297                         RTE_LOG(ERR, EAL,
4298                                 "fail  to register device event callback\n");
4299                         return -1;
4300                 }
4301         }
4302
4303         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4304                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4305
4306         /* set all ports to promiscuous mode by default */
4307         RTE_ETH_FOREACH_DEV(port_id) {
4308                 ret = rte_eth_promiscuous_enable(port_id);
4309                 if (ret != 0)
4310                         fprintf(stderr,
4311                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4312                                 port_id, rte_strerror(-ret));
4313         }
4314
4315 #ifdef RTE_LIB_METRICS
4316         /* Init metrics library */
4317         rte_metrics_init(rte_socket_id());
4318 #endif
4319
4320 #ifdef RTE_LIB_LATENCYSTATS
4321         if (latencystats_enabled != 0) {
4322                 int ret = rte_latencystats_init(1, NULL);
4323                 if (ret)
4324                         fprintf(stderr,
4325                                 "Warning: latencystats init() returned error %d\n",
4326                                 ret);
4327                 fprintf(stderr, "Latencystats running on lcore %d\n",
4328                         latencystats_lcore_id);
4329         }
4330 #endif
4331
4332         /* Setup bitrate stats */
4333 #ifdef RTE_LIB_BITRATESTATS
4334         if (bitrate_enabled != 0) {
4335                 bitrate_data = rte_stats_bitrate_create();
4336                 if (bitrate_data == NULL)
4337                         rte_exit(EXIT_FAILURE,
4338                                 "Could not allocate bitrate data.\n");
4339                 rte_stats_bitrate_reg(bitrate_data);
4340         }
4341 #endif
4342 #ifdef RTE_LIB_CMDLINE
4343         if (strlen(cmdline_filename) != 0)
4344                 cmdline_read_from_file(cmdline_filename);
4345
4346         if (interactive == 1) {
4347                 if (auto_start) {
4348                         printf("Start automatic packet forwarding\n");
4349                         start_packet_forwarding(0);
4350                 }
4351                 prompt();
4352                 pmd_test_exit();
4353         } else
4354 #endif
4355         {
4356                 char c;
4357                 int rc;
4358
4359                 f_quit = 0;
4360
4361                 printf("No commandline core given, start packet forwarding\n");
4362                 start_packet_forwarding(tx_first);
4363                 if (stats_period != 0) {
4364                         uint64_t prev_time = 0, cur_time, diff_time = 0;
4365                         uint64_t timer_period;
4366
4367                         /* Convert to number of cycles */
4368                         timer_period = stats_period * rte_get_timer_hz();
4369
4370                         while (f_quit == 0) {
4371                                 cur_time = rte_get_timer_cycles();
4372                                 diff_time += cur_time - prev_time;
4373
4374                                 if (diff_time >= timer_period) {
4375                                         print_stats();
4376                                         /* Reset the timer */
4377                                         diff_time = 0;
4378                                 }
4379                                 /* Sleep to avoid unnecessary checks */
4380                                 prev_time = cur_time;
4381                                 rte_delay_us_sleep(US_PER_S);
4382                         }
4383                 }
4384
4385                 printf("Press enter to exit\n");
4386                 rc = read(0, &c, 1);
4387                 pmd_test_exit();
4388                 if (rc < 0)
4389                         return 1;
4390         }
4391
4392         ret = rte_eal_cleanup();
4393         if (ret != 0)
4394                 rte_exit(EXIT_FAILURE,
4395                          "EAL cleanup failed: %s\n", strerror(-ret));
4396
4397         return EXIT_SUCCESS;
4398 }