app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_NET_IXGBE
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIB_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #ifdef RTE_LIB_METRICS
  58 #include <rte_metrics.h>
  59 #endif
  60 #ifdef RTE_LIB_BITRATESTATS
  61 #include <rte_bitrate.h>
  62 #endif
  63 #ifdef RTE_LIB_LATENCYSTATS
  64 #include <rte_latencystats.h>
  65 #endif
  66 #ifdef RTE_EXEC_ENV_WINDOWS
  67 #include <process.h>
  68 #endif
  69 #ifdef RTE_NET_BOND
  70 #include <rte_eth_bond.h>
  71 #endif
  72
  73 #include "testpmd.h"
  74
  75 #ifndef MAP_HUGETLB
  76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  77 #define HUGE_FLAG (0x40000)
  78 #else
  79 #define HUGE_FLAG MAP_HUGETLB
  80 #endif
  81
  82 #ifndef MAP_HUGE_SHIFT
  83 /* older kernels (or FreeBSD) will not have this define */
  84 #define HUGE_SHIFT (26)
  85 #else
  86 #define HUGE_SHIFT MAP_HUGE_SHIFT
  87 #endif
  88
  89 #define EXTMEM_HEAP_NAME "extmem"
  90 /*
  91  * Zone size with the malloc overhead (max of debug and release variants)
  92  * must fit into the smallest supported hugepage size (2M),
  93  * so that an IOVA-contiguous zone of this size can always be allocated
  94  * if there are free 2M hugepages.
  95  */
  96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE)
  97
  98 uint16_t verbose_level = 0; /**< Silent by default. */
  99 int testpmd_logtype; /**< Log type for testpmd logs */
 100
 101 /* use main core for command line ? */
 102 uint8_t interactive = 0;
 103 uint8_t auto_start = 0;
 104 uint8_t tx_first;
 105 char cmdline_filename[PATH_MAX] = {0};
 106
 107 /*
 108  * NUMA support configuration.
 109  * When set, the NUMA support attempts to dispatch the allocation of the
 110  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 111  * probed ports among the CPU sockets 0 and 1.
 112  * Otherwise, all memory is allocated from CPU socket 0.
 113  */
 114 uint8_t numa_support = 1; /**< numa enabled by default */
 115
 116 /*
 117  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 118  * not configured.
 119  */
 120 uint8_t socket_num = UMA_NO_CONFIG;
 121
 122 /*
 123  * Select mempool allocation type:
 124  * - native: use regular DPDK memory
 125  * - anon: use regular DPDK memory to create mempool, but populate using
 126  *         anonymous memory (may not be IOVA-contiguous)
 127  * - xmem: use externally allocated hugepage memory
 128  */
 129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 130
 131 /*
 132  * Store specified sockets on which memory pool to be used by ports
 133  * is allocated.
 134  */
 135 uint8_t port_numa[RTE_MAX_ETHPORTS];
 136
 137 /*
 138  * Store specified sockets on which RX ring to be used by ports
 139  * is allocated.
 140  */
 141 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 142
 143 /*
 144  * Store specified sockets on which TX ring to be used by ports
 145  * is allocated.
 146  */
 147 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 148
 149 /*
 150  * Record the Ethernet address of peer target ports to which packets are
 151  * forwarded.
 152  * Must be instantiated with the ethernet addresses of peer traffic generator
 153  * ports.
 154  */
 155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 156 portid_t nb_peer_eth_addrs = 0;
 157
 158 /*
 159  * Probed Target Environment.
 160  */
 161 struct rte_port *ports;        /**< For all probed ethernet ports. */
 162 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 164 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 165
 166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 167
 168 /*
 169  * Test Forwarding Configuration.
 170  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 171  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 172  */
 173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 175 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 176 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 177
 178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 180
 181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 182 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 183
 184 /*
 185  * Forwarding engines.
 186  */
 187 struct fwd_engine * fwd_engines[] = {
 188         &io_fwd_engine,
 189         &mac_fwd_engine,
 190         &mac_swap_engine,
 191         &flow_gen_engine,
 192         &rx_only_engine,
 193         &tx_only_engine,
 194         &csum_fwd_engine,
 195         &icmp_echo_engine,
 196         &noisy_vnf_engine,
 197         &five_tuple_swap_fwd_engine,
 198 #ifdef RTE_LIBRTE_IEEE1588
 199         &ieee1588_fwd_engine,
 200 #endif
 201         &shared_rxq_engine,
 202         NULL,
 203 };
 204
 205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 206 uint16_t mempool_flags;
 207
 208 struct fwd_config cur_fwd_config;
 209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 210 uint32_t retry_enabled;
 211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 213
 214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 216         DEFAULT_MBUF_DATA_SIZE
 217 }; /**< Mbuf data space size. */
 218 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 219                                       * specified on command-line. */
 220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 221
 222 /** Extended statistics to show. */
 223 struct rte_eth_xstat_name *xstats_display;
 224
 225 unsigned int xstats_display_num; /**< Size of extended statistics to show */
 226
 227 /*
 228  * In container, it cannot terminate the process which running with 'stats-period'
 229  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 230  */
 231 uint8_t f_quit;
 232
 233 /*
 234  * Max Rx frame size, set by '--max-pkt-len' parameter.
 235  */
 236 uint32_t max_rx_pkt_len;
 237
 238 /*
 239  * Configuration of packet segments used to scatter received packets
 240  * if some of split features is configured.
 241  */
 242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 243 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 245 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 246
 247 /*
 248  * Configuration of packet segments used by the "txonly" processing engine.
 249  */
 250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 252         TXONLY_DEF_PACKET_LEN,
 253 };
 254 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 255
 256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 257 /**< Split policy for packets to TX. */
 258
 259 uint8_t txonly_multi_flow;
 260 /**< Whether multiple flows are generated in TXONLY mode. */
 261
 262 uint32_t tx_pkt_times_inter;
 263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 264
 265 uint32_t tx_pkt_times_intra;
 266 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 267
 268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
 271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 272
 273 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 274 uint8_t dcb_config = 0;
 275
 276 /*
 277  * Configurable number of RX/TX queues.
 278  */
 279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 282
 283 /*
 284  * Configurable number of RX/TX ring descriptors.
 285  * Defaults are supplied by drivers via ethdev.
 286  */
 287 #define RTE_TEST_RX_DESC_DEFAULT 0
 288 #define RTE_TEST_TX_DESC_DEFAULT 0
 289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 291
 292 #define RTE_PMD_PARAM_UNSET -1
 293 /*
 294  * Configurable values of RX and TX ring threshold registers.
 295  */
 296
 297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 300
 301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 304
 305 /*
 306  * Configurable value of RX free threshold.
 307  */
 308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 309
 310 /*
 311  * Configurable value of RX drop enable.
 312  */
 313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 314
 315 /*
 316  * Configurable value of TX free threshold.
 317  */
 318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 319
 320 /*
 321  * Configurable value of TX RS bit threshold.
 322  */
 323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 324
 325 /*
 326  * Configurable value of buffered packets before sending.
 327  */
 328 uint16_t noisy_tx_sw_bufsz;
 329
 330 /*
 331  * Configurable value of packet buffer timeout.
 332  */
 333 uint16_t noisy_tx_sw_buf_flush_time;
 334
 335 /*
 336  * Configurable value for size of VNF internal memory area
 337  * used for simulating noisy neighbour behaviour
 338  */
 339 uint64_t noisy_lkup_mem_sz;
 340
 341 /*
 342  * Configurable value of number of random writes done in
 343  * VNF simulation memory area.
 344  */
 345 uint64_t noisy_lkup_num_writes;
 346
 347 /*
 348  * Configurable value of number of random reads done in
 349  * VNF simulation memory area.
 350  */
 351 uint64_t noisy_lkup_num_reads;
 352
 353 /*
 354  * Configurable value of number of random reads/writes done in
 355  * VNF simulation memory area.
 356  */
 357 uint64_t noisy_lkup_num_reads_writes;
 358
 359 /*
 360  * Receive Side Scaling (RSS) configuration.
 361  */
 362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */
 363
 364 /*
 365  * Port topology configuration
 366  */
 367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 368
 369 /*
 370  * Avoids to flush all the RX streams before starts forwarding.
 371  */
 372 uint8_t no_flush_rx = 0; /* flush by default */
 373
 374 /*
 375  * Flow API isolated mode.
 376  */
 377 uint8_t flow_isolate_all;
 378
 379 /*
 380  * Avoids to check link status when starting/stopping a port.
 381  */
 382 uint8_t no_link_check = 0; /* check by default */
 383
 384 /*
 385  * Don't automatically start all ports in interactive mode.
 386  */
 387 uint8_t no_device_start = 0;
 388
 389 /*
 390  * Enable link status change notification
 391  */
 392 uint8_t lsc_interrupt = 1; /* enabled by default */
 393
 394 /*
 395  * Enable device removal notification.
 396  */
 397 uint8_t rmv_interrupt = 1; /* enabled by default */
 398
 399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 400
 401 /* After attach, port setup is called on event or by iterator */
 402 bool setup_on_probe_event = true;
 403
 404 /* Clear ptypes on port initialization. */
 405 uint8_t clear_ptypes = true;
 406
 407 /* Hairpin ports configuration mode. */
 408 uint16_t hairpin_mode;
 409
 410 /* Pretty printing of ethdev events */
 411 static const char * const eth_event_desc[] = {
 412         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 413         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 414         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 415         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 416         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 417         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 418         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 419         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 420         [RTE_ETH_EVENT_NEW] = "device probed",
 421         [RTE_ETH_EVENT_DESTROY] = "device released",
 422         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 423         [RTE_ETH_EVENT_MAX] = NULL,
 424 };
 425
 426 /*
 427  * Display or mask ether events
 428  * Default to all events except VF_MBOX
 429  */
 430 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 431                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 432                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 433                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 434                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 435                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 436                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 437                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 438 /*
 439  * Decide if all memory are locked for performance.
 440  */
 441 int do_mlockall = 0;
 442
 443 /*
 444  * NIC bypass mode configuration options.
 445  */
 446
 447 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 448 /* The NIC bypass watchdog timeout. */
 449 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 450 #endif
 451
 452
 453 #ifdef RTE_LIB_LATENCYSTATS
 454
 455 /*
 456  * Set when latency stats is enabled in the commandline
 457  */
 458 uint8_t latencystats_enabled;
 459
 460 /*
 461  * Lcore ID to service latency statistics.
 462  */
 463 lcoreid_t latencystats_lcore_id = -1;
 464
 465 #endif
 466
 467 /*
 468  * Ethernet device configuration.
 469  */
 470 struct rte_eth_rxmode rx_mode;
 471
 472 struct rte_eth_txmode tx_mode = {
 473         .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
 474 };
 475
 476 struct rte_eth_fdir_conf fdir_conf = {
 477         .mode = RTE_FDIR_MODE_NONE,
 478         .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
 479         .status = RTE_FDIR_REPORT_STATUS,
 480         .mask = {
 481                 .vlan_tci_mask = 0xFFEF,
 482                 .ipv4_mask     = {
 483                         .src_ip = 0xFFFFFFFF,
 484                         .dst_ip = 0xFFFFFFFF,
 485                 },
 486                 .ipv6_mask     = {
 487                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 488                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 489                 },
 490                 .src_port_mask = 0xFFFF,
 491                 .dst_port_mask = 0xFFFF,
 492                 .mac_addr_byte_mask = 0xFF,
 493                 .tunnel_type_mask = 1,
 494                 .tunnel_id_mask = 0xFFFFFFFF,
 495         },
 496         .drop_queue = 127,
 497 };
 498
 499 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 500
 501 /*
 502  * Display zero values by default for xstats
 503  */
 504 uint8_t xstats_hide_zero;
 505
 506 /*
 507  * Measure of CPU cycles disabled by default
 508  */
 509 uint8_t record_core_cycles;
 510
 511 /*
 512  * Display of RX and TX bursts disabled by default
 513  */
 514 uint8_t record_burst_stats;
 515
 516 /*
 517  * Number of ports per shared Rx queue group, 0 disable.
 518  */
 519 uint32_t rxq_share;
 520
 521 unsigned int num_sockets = 0;
 522 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 523
 524 #ifdef RTE_LIB_BITRATESTATS
 525 /* Bitrate statistics */
 526 struct rte_stats_bitrates *bitrate_data;
 527 lcoreid_t bitrate_lcore_id;
 528 uint8_t bitrate_enabled;
 529 #endif
 530
 531 #ifdef RTE_LIB_GRO
 532 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 533 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 534 #endif
 535
 536 /*
 537  * hexadecimal bitmask of RX mq mode can be enabled.
 538  */
 539 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
 540
 541 /*
 542  * Used to set forced link speed
 543  */
 544 uint32_t eth_link_speed;
 545
 546 /*
 547  * ID of the current process in multi-process, used to
 548  * configure the queues to be polled.
 549  */
 550 int proc_id;
 551
 552 /*
 553  * Number of processes in multi-process, used to
 554  * configure the queues to be polled.
 555  */
 556 unsigned int num_procs = 1;
 557
 558 static void
 559 eth_rx_metadata_negotiate_mp(uint16_t port_id)
 560 {
 561         uint64_t rx_meta_features = 0;
 562         int ret;
 563
 564         if (!is_proc_primary())
 565                 return;
 566
 567         rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
 568         rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
 569         rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
 570
 571         ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
 572         if (ret == 0) {
 573                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
 574                         TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
 575                                     port_id);
 576                 }
 577
 578                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
 579                         TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
 580                                     port_id);
 581                 }
 582
 583                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
 584                         TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
 585                                     port_id);
 586                 }
 587         } else if (ret != -ENOTSUP) {
 588                 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
 589                          port_id, rte_strerror(-ret));
 590         }
 591 }
 592
 593 static int
 594 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 595                       const struct rte_eth_conf *dev_conf)
 596 {
 597         if (is_proc_primary())
 598                 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
 599                                         dev_conf);
 600         return 0;
 601 }
 602
 603 static int
 604 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop)
 605 {
 606 #ifdef RTE_NET_BOND
 607
 608         portid_t slave_pids[RTE_MAX_ETHPORTS];
 609         struct rte_port *port;
 610         int num_slaves;
 611         portid_t slave_pid;
 612         int i;
 613
 614         num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids,
 615                                                 RTE_MAX_ETHPORTS);
 616         if (num_slaves < 0) {
 617                 fprintf(stderr, "Failed to get slave list for port = %u\n",
 618                         bond_pid);
 619                 return num_slaves;
 620         }
 621
 622         for (i = 0; i < num_slaves; i++) {
 623                 slave_pid = slave_pids[i];
 624                 port = &ports[slave_pid];
 625                 port->port_status =
 626                         is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED;
 627         }
 628 #else
 629         RTE_SET_USED(bond_pid);
 630         RTE_SET_USED(is_stop);
 631 #endif
 632         return 0;
 633 }
 634
 635 static int
 636 eth_dev_start_mp(uint16_t port_id)
 637 {
 638         int ret;
 639
 640         if (is_proc_primary()) {
 641                 ret = rte_eth_dev_start(port_id);
 642                 if (ret != 0)
 643                         return ret;
 644
 645                 struct rte_port *port = &ports[port_id];
 646
 647                 /*
 648                  * Starting a bonded port also starts all slaves under the bonded
 649                  * device. So if this port is bond device, we need to modify the
 650                  * port status of these slaves.
 651                  */
 652                 if (port->bond_flag == 1)
 653                         return change_bonding_slave_port_status(port_id, false);
 654         }
 655
 656         return 0;
 657 }
 658
 659 static int
 660 eth_dev_stop_mp(uint16_t port_id)
 661 {
 662         int ret;
 663
 664         if (is_proc_primary()) {
 665                 ret = rte_eth_dev_stop(port_id);
 666                 if (ret != 0)
 667                         return ret;
 668
 669                 struct rte_port *port = &ports[port_id];
 670
 671                 /*
 672                  * Stopping a bonded port also stops all slaves under the bonded
 673                  * device. So if this port is bond device, we need to modify the
 674                  * port status of these slaves.
 675                  */
 676                 if (port->bond_flag == 1)
 677                         return change_bonding_slave_port_status(port_id, true);
 678         }
 679
 680         return 0;
 681 }
 682
 683 static void
 684 mempool_free_mp(struct rte_mempool *mp)
 685 {
 686         if (is_proc_primary())
 687                 rte_mempool_free(mp);
 688 }
 689
 690 static int
 691 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
 692 {
 693         if (is_proc_primary())
 694                 return rte_eth_dev_set_mtu(port_id, mtu);
 695
 696         return 0;
 697 }
 698
 699 /* Forward function declarations */
 700 static void setup_attached_port(portid_t pi);
 701 static void check_all_ports_link_status(uint32_t port_mask);
 702 static int eth_event_callback(portid_t port_id,
 703                               enum rte_eth_event_type type,
 704                               void *param, void *ret_param);
 705 static void dev_event_callback(const char *device_name,
 706                                 enum rte_dev_event_type type,
 707                                 void *param);
 708 static void fill_xstats_display_info(void);
 709
 710 /*
 711  * Check if all the ports are started.
 712  * If yes, return positive value. If not, return zero.
 713  */
 714 static int all_ports_started(void);
 715
 716 #ifdef RTE_LIB_GSO
 717 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 718 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 719 #endif
 720
 721 /* Holds the registered mbuf dynamic flags names. */
 722 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 723
 724
 725 /*
 726  * Helper function to check if socket is already discovered.
 727  * If yes, return positive value. If not, return zero.
 728  */
 729 int
 730 new_socket_id(unsigned int socket_id)
 731 {
 732         unsigned int i;
 733
 734         for (i = 0; i < num_sockets; i++) {
 735                 if (socket_ids[i] == socket_id)
 736                         return 0;
 737         }
 738         return 1;
 739 }
 740
 741 /*
 742  * Setup default configuration.
 743  */
 744 static void
 745 set_default_fwd_lcores_config(void)
 746 {
 747         unsigned int i;
 748         unsigned int nb_lc;
 749         unsigned int sock_num;
 750
 751         nb_lc = 0;
 752         for (i = 0; i < RTE_MAX_LCORE; i++) {
 753                 if (!rte_lcore_is_enabled(i))
 754                         continue;
 755                 sock_num = rte_lcore_to_socket_id(i);
 756                 if (new_socket_id(sock_num)) {
 757                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 758                                 rte_exit(EXIT_FAILURE,
 759                                          "Total sockets greater than %u\n",
 760                                          RTE_MAX_NUMA_NODES);
 761                         }
 762                         socket_ids[num_sockets++] = sock_num;
 763                 }
 764                 if (i == rte_get_main_lcore())
 765                         continue;
 766                 fwd_lcores_cpuids[nb_lc++] = i;
 767         }
 768         nb_lcores = (lcoreid_t) nb_lc;
 769         nb_cfg_lcores = nb_lcores;
 770         nb_fwd_lcores = 1;
 771 }
 772
 773 static void
 774 set_def_peer_eth_addrs(void)
 775 {
 776         portid_t i;
 777
 778         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 779                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 780                 peer_eth_addrs[i].addr_bytes[5] = i;
 781         }
 782 }
 783
 784 static void
 785 set_default_fwd_ports_config(void)
 786 {
 787         portid_t pt_id;
 788         int i = 0;
 789
 790         RTE_ETH_FOREACH_DEV(pt_id) {
 791                 fwd_ports_ids[i++] = pt_id;
 792
 793                 /* Update sockets info according to the attached device */
 794                 int socket_id = rte_eth_dev_socket_id(pt_id);
 795                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 796                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 797                                 rte_exit(EXIT_FAILURE,
 798                                          "Total sockets greater than %u\n",
 799                                          RTE_MAX_NUMA_NODES);
 800                         }
 801                         socket_ids[num_sockets++] = socket_id;
 802                 }
 803         }
 804
 805         nb_cfg_ports = nb_ports;
 806         nb_fwd_ports = nb_ports;
 807 }
 808
 809 void
 810 set_def_fwd_config(void)
 811 {
 812         set_default_fwd_lcores_config();
 813         set_def_peer_eth_addrs();
 814         set_default_fwd_ports_config();
 815 }
 816
 817 #ifndef RTE_EXEC_ENV_WINDOWS
 818 /* extremely pessimistic estimation of memory required to create a mempool */
 819 static int
 820 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 821 {
 822         unsigned int n_pages, mbuf_per_pg, leftover;
 823         uint64_t total_mem, mbuf_mem, obj_sz;
 824
 825         /* there is no good way to predict how much space the mempool will
 826          * occupy because it will allocate chunks on the fly, and some of those
 827          * will come from default DPDK memory while some will come from our
 828          * external memory, so just assume 128MB will be enough for everyone.
 829          */
 830         uint64_t hdr_mem = 128 << 20;
 831
 832         /* account for possible non-contiguousness */
 833         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 834         if (obj_sz > pgsz) {
 835                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 836                 return -1;
 837         }
 838
 839         mbuf_per_pg = pgsz / obj_sz;
 840         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 841         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 842
 843         mbuf_mem = n_pages * pgsz;
 844
 845         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 846
 847         if (total_mem > SIZE_MAX) {
 848                 TESTPMD_LOG(ERR, "Memory size too big\n");
 849                 return -1;
 850         }
 851         *out = (size_t)total_mem;
 852
 853         return 0;
 854 }
 855
 856 static int
 857 pagesz_flags(uint64_t page_sz)
 858 {
 859         /* as per mmap() manpage, all page sizes are log2 of page size
 860          * shifted by MAP_HUGE_SHIFT
 861          */
 862         int log2 = rte_log2_u64(page_sz);
 863
 864         return (log2 << HUGE_SHIFT);
 865 }
 866
 867 static void *
 868 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 869 {
 870         void *addr;
 871         int flags;
 872
 873         /* allocate anonymous hugepages */
 874         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 875         if (huge)
 876                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 877
 878         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 879         if (addr == MAP_FAILED)
 880                 return NULL;
 881
 882         return addr;
 883 }
 884
 885 struct extmem_param {
 886         void *addr;
 887         size_t len;
 888         size_t pgsz;
 889         rte_iova_t *iova_table;
 890         unsigned int iova_table_len;
 891 };
 892
 893 static int
 894 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 895                 bool huge)
 896 {
 897         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 898                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 899         unsigned int cur_page, n_pages, pgsz_idx;
 900         size_t mem_sz, cur_pgsz;
 901         rte_iova_t *iovas = NULL;
 902         void *addr;
 903         int ret;
 904
 905         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 906                 /* skip anything that is too big */
 907                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 908                         continue;
 909
 910                 cur_pgsz = pgsizes[pgsz_idx];
 911
 912                 /* if we were told not to allocate hugepages, override */
 913                 if (!huge)
 914                         cur_pgsz = sysconf(_SC_PAGESIZE);
 915
 916                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 917                 if (ret < 0) {
 918                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 919                         return -1;
 920                 }
 921
 922                 /* allocate our memory */
 923                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 924
 925                 /* if we couldn't allocate memory with a specified page size,
 926                  * that doesn't mean we can't do it with other page sizes, so
 927                  * try another one.
 928                  */
 929                 if (addr == NULL)
 930                         continue;
 931
 932                 /* store IOVA addresses for every page in this memory area */
 933                 n_pages = mem_sz / cur_pgsz;
 934
 935                 iovas = malloc(sizeof(*iovas) * n_pages);
 936
 937                 if (iovas == NULL) {
 938                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 939                         goto fail;
 940                 }
 941                 /* lock memory if it's not huge pages */
 942                 if (!huge)
 943                         mlock(addr, mem_sz);
 944
 945                 /* populate IOVA addresses */
 946                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 947                         rte_iova_t iova;
 948                         size_t offset;
 949                         void *cur;
 950
 951                         offset = cur_pgsz * cur_page;
 952                         cur = RTE_PTR_ADD(addr, offset);
 953
 954                         /* touch the page before getting its IOVA */
 955                         *(volatile char *)cur = 0;
 956
 957                         iova = rte_mem_virt2iova(cur);
 958
 959                         iovas[cur_page] = iova;
 960                 }
 961
 962                 break;
 963         }
 964         /* if we couldn't allocate anything */
 965         if (iovas == NULL)
 966                 return -1;
 967
 968         param->addr = addr;
 969         param->len = mem_sz;
 970         param->pgsz = cur_pgsz;
 971         param->iova_table = iovas;
 972         param->iova_table_len = n_pages;
 973
 974         return 0;
 975 fail:
 976         free(iovas);
 977         if (addr)
 978                 munmap(addr, mem_sz);
 979
 980         return -1;
 981 }
 982
 983 static int
 984 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 985 {
 986         struct extmem_param param;
 987         int socket_id, ret;
 988
 989         memset(&param, 0, sizeof(param));
 990
 991         /* check if our heap exists */
 992         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 993         if (socket_id < 0) {
 994                 /* create our heap */
 995                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 996                 if (ret < 0) {
 997                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 998                         return -1;
 999                 }
1000         }
1001
1002         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
1003         if (ret < 0) {
1004                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
1005                 return -1;
1006         }
1007
1008         /* we now have a valid memory area, so add it to heap */
1009         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
1010                         param.addr, param.len, param.iova_table,
1011                         param.iova_table_len, param.pgsz);
1012
1013         /* when using VFIO, memory is automatically mapped for DMA by EAL */
1014
1015         /* not needed any more */
1016         free(param.iova_table);
1017
1018         if (ret < 0) {
1019                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
1020                 munmap(param.addr, param.len);
1021                 return -1;
1022         }
1023
1024         /* success */
1025
1026         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
1027                         param.len >> 20);
1028
1029         return 0;
1030 }
1031 static void
1032 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1033              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1034 {
1035         uint16_t pid = 0;
1036         int ret;
1037
1038         RTE_ETH_FOREACH_DEV(pid) {
1039                 struct rte_eth_dev_info dev_info;
1040
1041                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1042                 if (ret != 0) {
1043                         TESTPMD_LOG(DEBUG,
1044                                     "unable to get device info for port %d on addr 0x%p,"
1045                                     "mempool unmapping will not be performed\n",
1046                                     pid, memhdr->addr);
1047                         continue;
1048                 }
1049
1050                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
1051                 if (ret) {
1052                         TESTPMD_LOG(DEBUG,
1053                                     "unable to DMA unmap addr 0x%p "
1054                                     "for device %s\n",
1055                                     memhdr->addr, dev_info.device->name);
1056                 }
1057         }
1058         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
1059         if (ret) {
1060                 TESTPMD_LOG(DEBUG,
1061                             "unable to un-register addr 0x%p\n", memhdr->addr);
1062         }
1063 }
1064
1065 static void
1066 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1067            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1068 {
1069         uint16_t pid = 0;
1070         size_t page_size = sysconf(_SC_PAGESIZE);
1071         int ret;
1072
1073         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1074                                   page_size);
1075         if (ret) {
1076                 TESTPMD_LOG(DEBUG,
1077                             "unable to register addr 0x%p\n", memhdr->addr);
1078                 return;
1079         }
1080         RTE_ETH_FOREACH_DEV(pid) {
1081                 struct rte_eth_dev_info dev_info;
1082
1083                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1084                 if (ret != 0) {
1085                         TESTPMD_LOG(DEBUG,
1086                                     "unable to get device info for port %d on addr 0x%p,"
1087                                     "mempool mapping will not be performed\n",
1088                                     pid, memhdr->addr);
1089                         continue;
1090                 }
1091                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1092                 if (ret) {
1093                         TESTPMD_LOG(DEBUG,
1094                                     "unable to DMA map addr 0x%p "
1095                                     "for device %s\n",
1096                                     memhdr->addr, dev_info.device->name);
1097                 }
1098         }
1099 }
1100 #endif
1101
1102 static unsigned int
1103 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1104             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1105 {
1106         struct rte_pktmbuf_extmem *xmem;
1107         unsigned int ext_num, zone_num, elt_num;
1108         uint16_t elt_size;
1109
1110         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1111         elt_num = EXTBUF_ZONE_SIZE / elt_size;
1112         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1113
1114         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1115         if (xmem == NULL) {
1116                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1117                                  "external buffer descriptors\n");
1118                 *ext_mem = NULL;
1119                 return 0;
1120         }
1121         for (ext_num = 0; ext_num < zone_num; ext_num++) {
1122                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1123                 const struct rte_memzone *mz;
1124                 char mz_name[RTE_MEMZONE_NAMESIZE];
1125                 int ret;
1126
1127                 ret = snprintf(mz_name, sizeof(mz_name),
1128                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1129                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1130                         errno = ENAMETOOLONG;
1131                         ext_num = 0;
1132                         break;
1133                 }
1134                 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE,
1135                                          socket_id,
1136                                          RTE_MEMZONE_IOVA_CONTIG |
1137                                          RTE_MEMZONE_1GB |
1138                                          RTE_MEMZONE_SIZE_HINT_ONLY);
1139                 if (mz == NULL) {
1140                         /*
1141                          * The caller exits on external buffer creation
1142                          * error, so there is no need to free memzones.
1143                          */
1144                         errno = ENOMEM;
1145                         ext_num = 0;
1146                         break;
1147                 }
1148                 xseg->buf_ptr = mz->addr;
1149                 xseg->buf_iova = mz->iova;
1150                 xseg->buf_len = EXTBUF_ZONE_SIZE;
1151                 xseg->elt_size = elt_size;
1152         }
1153         if (ext_num == 0 && xmem != NULL) {
1154                 free(xmem);
1155                 xmem = NULL;
1156         }
1157         *ext_mem = xmem;
1158         return ext_num;
1159 }
1160
1161 /*
1162  * Configuration initialisation done once at init time.
1163  */
1164 static struct rte_mempool *
1165 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1166                  unsigned int socket_id, uint16_t size_idx)
1167 {
1168         char pool_name[RTE_MEMPOOL_NAMESIZE];
1169         struct rte_mempool *rte_mp = NULL;
1170 #ifndef RTE_EXEC_ENV_WINDOWS
1171         uint32_t mb_size;
1172
1173         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1174 #endif
1175         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1176         if (!is_proc_primary()) {
1177                 rte_mp = rte_mempool_lookup(pool_name);
1178                 if (rte_mp == NULL)
1179                         rte_exit(EXIT_FAILURE,
1180                                 "Get mbuf pool for socket %u failed: %s\n",
1181                                 socket_id, rte_strerror(rte_errno));
1182                 return rte_mp;
1183         }
1184
1185         TESTPMD_LOG(INFO,
1186                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1187                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1188
1189         switch (mp_alloc_type) {
1190         case MP_ALLOC_NATIVE:
1191                 {
1192                         /* wrapper to rte_mempool_create() */
1193                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1194                                         rte_mbuf_best_mempool_ops());
1195                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1196                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1197                         break;
1198                 }
1199 #ifndef RTE_EXEC_ENV_WINDOWS
1200         case MP_ALLOC_ANON:
1201                 {
1202                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1203                                 mb_size, (unsigned int) mb_mempool_cache,
1204                                 sizeof(struct rte_pktmbuf_pool_private),
1205                                 socket_id, mempool_flags);
1206                         if (rte_mp == NULL)
1207                                 goto err;
1208
1209                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1210                                 rte_mempool_free(rte_mp);
1211                                 rte_mp = NULL;
1212                                 goto err;
1213                         }
1214                         rte_pktmbuf_pool_init(rte_mp, NULL);
1215                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1216                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1217                         break;
1218                 }
1219         case MP_ALLOC_XMEM:
1220         case MP_ALLOC_XMEM_HUGE:
1221                 {
1222                         int heap_socket;
1223                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1224
1225                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1226                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1227
1228                         heap_socket =
1229                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1230                         if (heap_socket < 0)
1231                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1232
1233                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1234                                         rte_mbuf_best_mempool_ops());
1235                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1236                                         mb_mempool_cache, 0, mbuf_seg_size,
1237                                         heap_socket);
1238                         break;
1239                 }
1240 #endif
1241         case MP_ALLOC_XBUF:
1242                 {
1243                         struct rte_pktmbuf_extmem *ext_mem;
1244                         unsigned int ext_num;
1245
1246                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1247                                                socket_id, pool_name, &ext_mem);
1248                         if (ext_num == 0)
1249                                 rte_exit(EXIT_FAILURE,
1250                                          "Can't create pinned data buffers\n");
1251
1252                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1253                                         rte_mbuf_best_mempool_ops());
1254                         rte_mp = rte_pktmbuf_pool_create_extbuf
1255                                         (pool_name, nb_mbuf, mb_mempool_cache,
1256                                          0, mbuf_seg_size, socket_id,
1257                                          ext_mem, ext_num);
1258                         free(ext_mem);
1259                         break;
1260                 }
1261         default:
1262                 {
1263                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1264                 }
1265         }
1266
1267 #ifndef RTE_EXEC_ENV_WINDOWS
1268 err:
1269 #endif
1270         if (rte_mp == NULL) {
1271                 rte_exit(EXIT_FAILURE,
1272                         "Creation of mbuf pool for socket %u failed: %s\n",
1273                         socket_id, rte_strerror(rte_errno));
1274         } else if (verbose_level > 0) {
1275                 rte_mempool_dump(stdout, rte_mp);
1276         }
1277         return rte_mp;
1278 }
1279
1280 /*
1281  * Check given socket id is valid or not with NUMA mode,
1282  * if valid, return 0, else return -1
1283  */
1284 static int
1285 check_socket_id(const unsigned int socket_id)
1286 {
1287         static int warning_once = 0;
1288
1289         if (new_socket_id(socket_id)) {
1290                 if (!warning_once && numa_support)
1291                         fprintf(stderr,
1292                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1293                 warning_once = 1;
1294                 return -1;
1295         }
1296         return 0;
1297 }
1298
1299 /*
1300  * Get the allowed maximum number of RX queues.
1301  * *pid return the port id which has minimal value of
1302  * max_rx_queues in all ports.
1303  */
1304 queueid_t
1305 get_allowed_max_nb_rxq(portid_t *pid)
1306 {
1307         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1308         bool max_rxq_valid = false;
1309         portid_t pi;
1310         struct rte_eth_dev_info dev_info;
1311
1312         RTE_ETH_FOREACH_DEV(pi) {
1313                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1314                         continue;
1315
1316                 max_rxq_valid = true;
1317                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1318                         allowed_max_rxq = dev_info.max_rx_queues;
1319                         *pid = pi;
1320                 }
1321         }
1322         return max_rxq_valid ? allowed_max_rxq : 0;
1323 }
1324
1325 /*
1326  * Check input rxq is valid or not.
1327  * If input rxq is not greater than any of maximum number
1328  * of RX queues of all ports, it is valid.
1329  * if valid, return 0, else return -1
1330  */
1331 int
1332 check_nb_rxq(queueid_t rxq)
1333 {
1334         queueid_t allowed_max_rxq;
1335         portid_t pid = 0;
1336
1337         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1338         if (rxq > allowed_max_rxq) {
1339                 fprintf(stderr,
1340                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1341                         rxq, allowed_max_rxq, pid);
1342                 return -1;
1343         }
1344         return 0;
1345 }
1346
1347 /*
1348  * Get the allowed maximum number of TX queues.
1349  * *pid return the port id which has minimal value of
1350  * max_tx_queues in all ports.
1351  */
1352 queueid_t
1353 get_allowed_max_nb_txq(portid_t *pid)
1354 {
1355         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1356         bool max_txq_valid = false;
1357         portid_t pi;
1358         struct rte_eth_dev_info dev_info;
1359
1360         RTE_ETH_FOREACH_DEV(pi) {
1361                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1362                         continue;
1363
1364                 max_txq_valid = true;
1365                 if (dev_info.max_tx_queues < allowed_max_txq) {
1366                         allowed_max_txq = dev_info.max_tx_queues;
1367                         *pid = pi;
1368                 }
1369         }
1370         return max_txq_valid ? allowed_max_txq : 0;
1371 }
1372
1373 /*
1374  * Check input txq is valid or not.
1375  * If input txq is not greater than any of maximum number
1376  * of TX queues of all ports, it is valid.
1377  * if valid, return 0, else return -1
1378  */
1379 int
1380 check_nb_txq(queueid_t txq)
1381 {
1382         queueid_t allowed_max_txq;
1383         portid_t pid = 0;
1384
1385         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1386         if (txq > allowed_max_txq) {
1387                 fprintf(stderr,
1388                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1389                         txq, allowed_max_txq, pid);
1390                 return -1;
1391         }
1392         return 0;
1393 }
1394
1395 /*
1396  * Get the allowed maximum number of RXDs of every rx queue.
1397  * *pid return the port id which has minimal value of
1398  * max_rxd in all queues of all ports.
1399  */
1400 static uint16_t
1401 get_allowed_max_nb_rxd(portid_t *pid)
1402 {
1403         uint16_t allowed_max_rxd = UINT16_MAX;
1404         portid_t pi;
1405         struct rte_eth_dev_info dev_info;
1406
1407         RTE_ETH_FOREACH_DEV(pi) {
1408                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1409                         continue;
1410
1411                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1412                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1413                         *pid = pi;
1414                 }
1415         }
1416         return allowed_max_rxd;
1417 }
1418
1419 /*
1420  * Get the allowed minimal number of RXDs of every rx queue.
1421  * *pid return the port id which has minimal value of
1422  * min_rxd in all queues of all ports.
1423  */
1424 static uint16_t
1425 get_allowed_min_nb_rxd(portid_t *pid)
1426 {
1427         uint16_t allowed_min_rxd = 0;
1428         portid_t pi;
1429         struct rte_eth_dev_info dev_info;
1430
1431         RTE_ETH_FOREACH_DEV(pi) {
1432                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1433                         continue;
1434
1435                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1436                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1437                         *pid = pi;
1438                 }
1439         }
1440
1441         return allowed_min_rxd;
1442 }
1443
1444 /*
1445  * Check input rxd is valid or not.
1446  * If input rxd is not greater than any of maximum number
1447  * of RXDs of every Rx queues and is not less than any of
1448  * minimal number of RXDs of every Rx queues, it is valid.
1449  * if valid, return 0, else return -1
1450  */
1451 int
1452 check_nb_rxd(queueid_t rxd)
1453 {
1454         uint16_t allowed_max_rxd;
1455         uint16_t allowed_min_rxd;
1456         portid_t pid = 0;
1457
1458         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1459         if (rxd > allowed_max_rxd) {
1460                 fprintf(stderr,
1461                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1462                         rxd, allowed_max_rxd, pid);
1463                 return -1;
1464         }
1465
1466         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1467         if (rxd < allowed_min_rxd) {
1468                 fprintf(stderr,
1469                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1470                         rxd, allowed_min_rxd, pid);
1471                 return -1;
1472         }
1473
1474         return 0;
1475 }
1476
1477 /*
1478  * Get the allowed maximum number of TXDs of every rx queues.
1479  * *pid return the port id which has minimal value of
1480  * max_txd in every tx queue.
1481  */
1482 static uint16_t
1483 get_allowed_max_nb_txd(portid_t *pid)
1484 {
1485         uint16_t allowed_max_txd = UINT16_MAX;
1486         portid_t pi;
1487         struct rte_eth_dev_info dev_info;
1488
1489         RTE_ETH_FOREACH_DEV(pi) {
1490                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1491                         continue;
1492
1493                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1494                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1495                         *pid = pi;
1496                 }
1497         }
1498         return allowed_max_txd;
1499 }
1500
1501 /*
1502  * Get the allowed maximum number of TXDs of every tx queues.
1503  * *pid return the port id which has minimal value of
1504  * min_txd in every tx queue.
1505  */
1506 static uint16_t
1507 get_allowed_min_nb_txd(portid_t *pid)
1508 {
1509         uint16_t allowed_min_txd = 0;
1510         portid_t pi;
1511         struct rte_eth_dev_info dev_info;
1512
1513         RTE_ETH_FOREACH_DEV(pi) {
1514                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1515                         continue;
1516
1517                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1518                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1519                         *pid = pi;
1520                 }
1521         }
1522
1523         return allowed_min_txd;
1524 }
1525
1526 /*
1527  * Check input txd is valid or not.
1528  * If input txd is not greater than any of maximum number
1529  * of TXDs of every Rx queues, it is valid.
1530  * if valid, return 0, else return -1
1531  */
1532 int
1533 check_nb_txd(queueid_t txd)
1534 {
1535         uint16_t allowed_max_txd;
1536         uint16_t allowed_min_txd;
1537         portid_t pid = 0;
1538
1539         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1540         if (txd > allowed_max_txd) {
1541                 fprintf(stderr,
1542                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1543                         txd, allowed_max_txd, pid);
1544                 return -1;
1545         }
1546
1547         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1548         if (txd < allowed_min_txd) {
1549                 fprintf(stderr,
1550                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1551                         txd, allowed_min_txd, pid);
1552                 return -1;
1553         }
1554         return 0;
1555 }
1556
1557
1558 /*
1559  * Get the allowed maximum number of hairpin queues.
1560  * *pid return the port id which has minimal value of
1561  * max_hairpin_queues in all ports.
1562  */
1563 queueid_t
1564 get_allowed_max_nb_hairpinq(portid_t *pid)
1565 {
1566         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1567         portid_t pi;
1568         struct rte_eth_hairpin_cap cap;
1569
1570         RTE_ETH_FOREACH_DEV(pi) {
1571                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1572                         *pid = pi;
1573                         return 0;
1574                 }
1575                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1576                         allowed_max_hairpinq = cap.max_nb_queues;
1577                         *pid = pi;
1578                 }
1579         }
1580         return allowed_max_hairpinq;
1581 }
1582
1583 /*
1584  * Check input hairpin is valid or not.
1585  * If input hairpin is not greater than any of maximum number
1586  * of hairpin queues of all ports, it is valid.
1587  * if valid, return 0, else return -1
1588  */
1589 int
1590 check_nb_hairpinq(queueid_t hairpinq)
1591 {
1592         queueid_t allowed_max_hairpinq;
1593         portid_t pid = 0;
1594
1595         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1596         if (hairpinq > allowed_max_hairpinq) {
1597                 fprintf(stderr,
1598                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1599                         hairpinq, allowed_max_hairpinq, pid);
1600                 return -1;
1601         }
1602         return 0;
1603 }
1604
1605 static int
1606 get_eth_overhead(struct rte_eth_dev_info *dev_info)
1607 {
1608         uint32_t eth_overhead;
1609
1610         if (dev_info->max_mtu != UINT16_MAX &&
1611             dev_info->max_rx_pktlen > dev_info->max_mtu)
1612                 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1613         else
1614                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1615
1616         return eth_overhead;
1617 }
1618
1619 static void
1620 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1621 {
1622         struct rte_port *port = &ports[pid];
1623         int ret;
1624         int i;
1625
1626         eth_rx_metadata_negotiate_mp(pid);
1627
1628         port->dev_conf.txmode = tx_mode;
1629         port->dev_conf.rxmode = rx_mode;
1630
1631         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1632         if (ret != 0)
1633                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1634
1635         if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1636                 port->dev_conf.txmode.offloads &=
1637                         ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1638
1639         /* Apply Rx offloads configuration */
1640         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1641                 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1642         /* Apply Tx offloads configuration */
1643         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1644                 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1645
1646         if (eth_link_speed)
1647                 port->dev_conf.link_speeds = eth_link_speed;
1648
1649         if (max_rx_pkt_len)
1650                 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1651                         get_eth_overhead(&port->dev_info);
1652
1653         /* set flag to initialize port/queue */
1654         port->need_reconfig = 1;
1655         port->need_reconfig_queues = 1;
1656         port->socket_id = socket_id;
1657         port->tx_metadata = 0;
1658
1659         /*
1660          * Check for maximum number of segments per MTU.
1661          * Accordingly update the mbuf data size.
1662          */
1663         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1664             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1665                 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1666                 uint16_t mtu;
1667
1668                 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1669                         uint16_t data_size = (mtu + eth_overhead) /
1670                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1671                         uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1672
1673                         if (buffer_size > mbuf_data_size[0]) {
1674                                 mbuf_data_size[0] = buffer_size;
1675                                 TESTPMD_LOG(WARNING,
1676                                         "Configured mbuf size of the first segment %hu\n",
1677                                         mbuf_data_size[0]);
1678                         }
1679                 }
1680         }
1681 }
1682
1683 static void
1684 init_config(void)
1685 {
1686         portid_t pid;
1687         struct rte_mempool *mbp;
1688         unsigned int nb_mbuf_per_pool;
1689         lcoreid_t  lc_id;
1690 #ifdef RTE_LIB_GRO
1691         struct rte_gro_param gro_param;
1692 #endif
1693 #ifdef RTE_LIB_GSO
1694         uint32_t gso_types;
1695 #endif
1696
1697         /* Configuration of logical cores. */
1698         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1699                                 sizeof(struct fwd_lcore *) * nb_lcores,
1700                                 RTE_CACHE_LINE_SIZE);
1701         if (fwd_lcores == NULL) {
1702                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1703                                                         "failed\n", nb_lcores);
1704         }
1705         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1706                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1707                                                sizeof(struct fwd_lcore),
1708                                                RTE_CACHE_LINE_SIZE);
1709                 if (fwd_lcores[lc_id] == NULL) {
1710                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1711                                                                 "failed\n");
1712                 }
1713                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1714         }
1715
1716         RTE_ETH_FOREACH_DEV(pid) {
1717                 uint32_t socket_id;
1718
1719                 if (numa_support) {
1720                         socket_id = port_numa[pid];
1721                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1722                                 socket_id = rte_eth_dev_socket_id(pid);
1723
1724                                 /*
1725                                  * if socket_id is invalid,
1726                                  * set to the first available socket.
1727                                  */
1728                                 if (check_socket_id(socket_id) < 0)
1729                                         socket_id = socket_ids[0];
1730                         }
1731                 } else {
1732                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1733                                     0 : socket_num;
1734                 }
1735                 /* Apply default TxRx configuration for all ports */
1736                 init_config_port_offloads(pid, socket_id);
1737         }
1738         /*
1739          * Create pools of mbuf.
1740          * If NUMA support is disabled, create a single pool of mbuf in
1741          * socket 0 memory by default.
1742          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1743          *
1744          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1745          * nb_txd can be configured at run time.
1746          */
1747         if (param_total_num_mbufs)
1748                 nb_mbuf_per_pool = param_total_num_mbufs;
1749         else {
1750                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1751                         (nb_lcores * mb_mempool_cache) +
1752                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1753                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1754         }
1755
1756         if (numa_support) {
1757                 uint8_t i, j;
1758
1759                 for (i = 0; i < num_sockets; i++)
1760                         for (j = 0; j < mbuf_data_size_n; j++)
1761                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1762                                         mbuf_pool_create(mbuf_data_size[j],
1763                                                           nb_mbuf_per_pool,
1764                                                           socket_ids[i], j);
1765         } else {
1766                 uint8_t i;
1767
1768                 for (i = 0; i < mbuf_data_size_n; i++)
1769                         mempools[i] = mbuf_pool_create
1770                                         (mbuf_data_size[i],
1771                                          nb_mbuf_per_pool,
1772                                          socket_num == UMA_NO_CONFIG ?
1773                                          0 : socket_num, i);
1774         }
1775
1776         init_port_config();
1777
1778 #ifdef RTE_LIB_GSO
1779         gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1780                 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1781 #endif
1782         /*
1783          * Records which Mbuf pool to use by each logical core, if needed.
1784          */
1785         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1786                 mbp = mbuf_pool_find(
1787                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1788
1789                 if (mbp == NULL)
1790                         mbp = mbuf_pool_find(0, 0);
1791                 fwd_lcores[lc_id]->mbp = mbp;
1792 #ifdef RTE_LIB_GSO
1793                 /* initialize GSO context */
1794                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1795                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1796                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1797                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1798                         RTE_ETHER_CRC_LEN;
1799                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1800 #endif
1801         }
1802
1803         fwd_config_setup();
1804
1805 #ifdef RTE_LIB_GRO
1806         /* create a gro context for each lcore */
1807         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1808         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1809         gro_param.max_item_per_flow = MAX_PKT_BURST;
1810         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1811                 gro_param.socket_id = rte_lcore_to_socket_id(
1812                                 fwd_lcores_cpuids[lc_id]);
1813                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1814                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1815                         rte_exit(EXIT_FAILURE,
1816                                         "rte_gro_ctx_create() failed\n");
1817                 }
1818         }
1819 #endif
1820 }
1821
1822
1823 void
1824 reconfig(portid_t new_port_id, unsigned socket_id)
1825 {
1826         /* Reconfiguration of Ethernet ports. */
1827         init_config_port_offloads(new_port_id, socket_id);
1828         init_port_config();
1829 }
1830
1831
1832 int
1833 init_fwd_streams(void)
1834 {
1835         portid_t pid;
1836         struct rte_port *port;
1837         streamid_t sm_id, nb_fwd_streams_new;
1838         queueid_t q;
1839
1840         /* set socket id according to numa or not */
1841         RTE_ETH_FOREACH_DEV(pid) {
1842                 port = &ports[pid];
1843                 if (nb_rxq > port->dev_info.max_rx_queues) {
1844                         fprintf(stderr,
1845                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1846                                 nb_rxq, port->dev_info.max_rx_queues);
1847                         return -1;
1848                 }
1849                 if (nb_txq > port->dev_info.max_tx_queues) {
1850                         fprintf(stderr,
1851                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1852                                 nb_txq, port->dev_info.max_tx_queues);
1853                         return -1;
1854                 }
1855                 if (numa_support) {
1856                         if (port_numa[pid] != NUMA_NO_CONFIG)
1857                                 port->socket_id = port_numa[pid];
1858                         else {
1859                                 port->socket_id = rte_eth_dev_socket_id(pid);
1860
1861                                 /*
1862                                  * if socket_id is invalid,
1863                                  * set to the first available socket.
1864                                  */
1865                                 if (check_socket_id(port->socket_id) < 0)
1866                                         port->socket_id = socket_ids[0];
1867                         }
1868                 }
1869                 else {
1870                         if (socket_num == UMA_NO_CONFIG)
1871                                 port->socket_id = 0;
1872                         else
1873                                 port->socket_id = socket_num;
1874                 }
1875         }
1876
1877         q = RTE_MAX(nb_rxq, nb_txq);
1878         if (q == 0) {
1879                 fprintf(stderr,
1880                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1881                 return -1;
1882         }
1883         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1884         if (nb_fwd_streams_new == nb_fwd_streams)
1885                 return 0;
1886         /* clear the old */
1887         if (fwd_streams != NULL) {
1888                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1889                         if (fwd_streams[sm_id] == NULL)
1890                                 continue;
1891                         rte_free(fwd_streams[sm_id]);
1892                         fwd_streams[sm_id] = NULL;
1893                 }
1894                 rte_free(fwd_streams);
1895                 fwd_streams = NULL;
1896         }
1897
1898         /* init new */
1899         nb_fwd_streams = nb_fwd_streams_new;
1900         if (nb_fwd_streams) {
1901                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1902                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1903                         RTE_CACHE_LINE_SIZE);
1904                 if (fwd_streams == NULL)
1905                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1906                                  " (struct fwd_stream *)) failed\n",
1907                                  nb_fwd_streams);
1908
1909                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1910                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1911                                 " struct fwd_stream", sizeof(struct fwd_stream),
1912                                 RTE_CACHE_LINE_SIZE);
1913                         if (fwd_streams[sm_id] == NULL)
1914                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1915                                          "(struct fwd_stream) failed\n");
1916                 }
1917         }
1918
1919         return 0;
1920 }
1921
1922 static void
1923 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1924 {
1925         uint64_t total_burst, sburst;
1926         uint64_t nb_burst;
1927         uint64_t burst_stats[4];
1928         uint16_t pktnb_stats[4];
1929         uint16_t nb_pkt;
1930         int burst_percent[4], sburstp;
1931         int i;
1932
1933         /*
1934          * First compute the total number of packet bursts and the
1935          * two highest numbers of bursts of the same number of packets.
1936          */
1937         memset(&burst_stats, 0x0, sizeof(burst_stats));
1938         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1939
1940         /* Show stats for 0 burst size always */
1941         total_burst = pbs->pkt_burst_spread[0];
1942         burst_stats[0] = pbs->pkt_burst_spread[0];
1943         pktnb_stats[0] = 0;
1944
1945         /* Find the next 2 burst sizes with highest occurrences. */
1946         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1947                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1948
1949                 if (nb_burst == 0)
1950                         continue;
1951
1952                 total_burst += nb_burst;
1953
1954                 if (nb_burst > burst_stats[1]) {
1955                         burst_stats[2] = burst_stats[1];
1956                         pktnb_stats[2] = pktnb_stats[1];
1957                         burst_stats[1] = nb_burst;
1958                         pktnb_stats[1] = nb_pkt;
1959                 } else if (nb_burst > burst_stats[2]) {
1960                         burst_stats[2] = nb_burst;
1961                         pktnb_stats[2] = nb_pkt;
1962                 }
1963         }
1964         if (total_burst == 0)
1965                 return;
1966
1967         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1968         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1969                 if (i == 3) {
1970                         printf("%d%% of other]\n", 100 - sburstp);
1971                         return;
1972                 }
1973
1974                 sburst += burst_stats[i];
1975                 if (sburst == total_burst) {
1976                         printf("%d%% of %d pkts]\n",
1977                                 100 - sburstp, (int) pktnb_stats[i]);
1978                         return;
1979                 }
1980
1981                 burst_percent[i] =
1982                         (double)burst_stats[i] / total_burst * 100;
1983                 printf("%d%% of %d pkts + ",
1984                         burst_percent[i], (int) pktnb_stats[i]);
1985                 sburstp += burst_percent[i];
1986         }
1987 }
1988
1989 static void
1990 fwd_stream_stats_display(streamid_t stream_id)
1991 {
1992         struct fwd_stream *fs;
1993         static const char *fwd_top_stats_border = "-------";
1994
1995         fs = fwd_streams[stream_id];
1996         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1997             (fs->fwd_dropped == 0))
1998                 return;
1999         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
2000                "TX Port=%2d/Queue=%2d %s\n",
2001                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
2002                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
2003         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
2004                " TX-dropped: %-14"PRIu64,
2005                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
2006
2007         /* if checksum mode */
2008         if (cur_fwd_eng == &csum_fwd_engine) {
2009                 printf("  RX- bad IP checksum: %-14"PRIu64
2010                        "  Rx- bad L4 checksum: %-14"PRIu64
2011                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
2012                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
2013                         fs->rx_bad_outer_l4_csum);
2014                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
2015                         fs->rx_bad_outer_ip_csum);
2016         } else {
2017                 printf("\n");
2018         }
2019
2020         if (record_burst_stats) {
2021                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
2022                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
2023         }
2024 }
2025
2026 void
2027 fwd_stats_display(void)
2028 {
2029         static const char *fwd_stats_border = "----------------------";
2030         static const char *acc_stats_border = "+++++++++++++++";
2031         struct {
2032                 struct fwd_stream *rx_stream;
2033                 struct fwd_stream *tx_stream;
2034                 uint64_t tx_dropped;
2035                 uint64_t rx_bad_ip_csum;
2036                 uint64_t rx_bad_l4_csum;
2037                 uint64_t rx_bad_outer_l4_csum;
2038                 uint64_t rx_bad_outer_ip_csum;
2039         } ports_stats[RTE_MAX_ETHPORTS];
2040         uint64_t total_rx_dropped = 0;
2041         uint64_t total_tx_dropped = 0;
2042         uint64_t total_rx_nombuf = 0;
2043         struct rte_eth_stats stats;
2044         uint64_t fwd_cycles = 0;
2045         uint64_t total_recv = 0;
2046         uint64_t total_xmit = 0;
2047         struct rte_port *port;
2048         streamid_t sm_id;
2049         portid_t pt_id;
2050         int ret;
2051         int i;
2052
2053         memset(ports_stats, 0, sizeof(ports_stats));
2054
2055         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2056                 struct fwd_stream *fs = fwd_streams[sm_id];
2057
2058                 if (cur_fwd_config.nb_fwd_streams >
2059                     cur_fwd_config.nb_fwd_ports) {
2060                         fwd_stream_stats_display(sm_id);
2061                 } else {
2062                         ports_stats[fs->tx_port].tx_stream = fs;
2063                         ports_stats[fs->rx_port].rx_stream = fs;
2064                 }
2065
2066                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
2067
2068                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
2069                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2070                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2071                                 fs->rx_bad_outer_l4_csum;
2072                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2073                                 fs->rx_bad_outer_ip_csum;
2074
2075                 if (record_core_cycles)
2076                         fwd_cycles += fs->core_cycles;
2077         }
2078         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2079                 pt_id = fwd_ports_ids[i];
2080                 port = &ports[pt_id];
2081
2082                 ret = rte_eth_stats_get(pt_id, &stats);
2083                 if (ret != 0) {
2084                         fprintf(stderr,
2085                                 "%s: Error: failed to get stats (port %u): %d",
2086                                 __func__, pt_id, ret);
2087                         continue;
2088                 }
2089                 stats.ipackets -= port->stats.ipackets;
2090                 stats.opackets -= port->stats.opackets;
2091                 stats.ibytes -= port->stats.ibytes;
2092                 stats.obytes -= port->stats.obytes;
2093                 stats.imissed -= port->stats.imissed;
2094                 stats.oerrors -= port->stats.oerrors;
2095                 stats.rx_nombuf -= port->stats.rx_nombuf;
2096
2097                 total_recv += stats.ipackets;
2098                 total_xmit += stats.opackets;
2099                 total_rx_dropped += stats.imissed;
2100                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2101                 total_tx_dropped += stats.oerrors;
2102                 total_rx_nombuf  += stats.rx_nombuf;
2103
2104                 printf("\n  %s Forward statistics for port %-2d %s\n",
2105                        fwd_stats_border, pt_id, fwd_stats_border);
2106
2107                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2108                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2109                        stats.ipackets + stats.imissed);
2110
2111                 if (cur_fwd_eng == &csum_fwd_engine) {
2112                         printf("  Bad-ipcsum: %-14"PRIu64
2113                                " Bad-l4csum: %-14"PRIu64
2114                                "Bad-outer-l4csum: %-14"PRIu64"\n",
2115                                ports_stats[pt_id].rx_bad_ip_csum,
2116                                ports_stats[pt_id].rx_bad_l4_csum,
2117                                ports_stats[pt_id].rx_bad_outer_l4_csum);
2118                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
2119                                ports_stats[pt_id].rx_bad_outer_ip_csum);
2120                 }
2121                 if (stats.ierrors + stats.rx_nombuf > 0) {
2122                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
2123                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2124                 }
2125
2126                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2127                        "TX-total: %-"PRIu64"\n",
2128                        stats.opackets, ports_stats[pt_id].tx_dropped,
2129                        stats.opackets + ports_stats[pt_id].tx_dropped);
2130
2131                 if (record_burst_stats) {
2132                         if (ports_stats[pt_id].rx_stream)
2133                                 pkt_burst_stats_display("RX",
2134                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
2135                         if (ports_stats[pt_id].tx_stream)
2136                                 pkt_burst_stats_display("TX",
2137                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2138                 }
2139
2140                 printf("  %s--------------------------------%s\n",
2141                        fwd_stats_border, fwd_stats_border);
2142         }
2143
2144         printf("\n  %s Accumulated forward statistics for all ports"
2145                "%s\n",
2146                acc_stats_border, acc_stats_border);
2147         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2148                "%-"PRIu64"\n"
2149                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2150                "%-"PRIu64"\n",
2151                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2152                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2153         if (total_rx_nombuf > 0)
2154                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2155         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
2156                "%s\n",
2157                acc_stats_border, acc_stats_border);
2158         if (record_core_cycles) {
2159 #define CYC_PER_MHZ 1E6
2160                 if (total_recv > 0 || total_xmit > 0) {
2161                         uint64_t total_pkts = 0;
2162                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2163                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2164                                 total_pkts = total_xmit;
2165                         else
2166                                 total_pkts = total_recv;
2167
2168                         printf("\n  CPU cycles/packet=%.2F (total cycles="
2169                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2170                                " MHz Clock\n",
2171                                (double) fwd_cycles / total_pkts,
2172                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2173                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2174                 }
2175         }
2176 }
2177
2178 void
2179 fwd_stats_reset(void)
2180 {
2181         streamid_t sm_id;
2182         portid_t pt_id;
2183         int ret;
2184         int i;
2185
2186         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2187                 pt_id = fwd_ports_ids[i];
2188                 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2189                 if (ret != 0)
2190                         fprintf(stderr,
2191                                 "%s: Error: failed to clear stats (port %u):%d",
2192                                 __func__, pt_id, ret);
2193         }
2194         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2195                 struct fwd_stream *fs = fwd_streams[sm_id];
2196
2197                 fs->rx_packets = 0;
2198                 fs->tx_packets = 0;
2199                 fs->fwd_dropped = 0;
2200                 fs->rx_bad_ip_csum = 0;
2201                 fs->rx_bad_l4_csum = 0;
2202                 fs->rx_bad_outer_l4_csum = 0;
2203                 fs->rx_bad_outer_ip_csum = 0;
2204
2205                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2206                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2207                 fs->core_cycles = 0;
2208         }
2209 }
2210
2211 static void
2212 flush_fwd_rx_queues(void)
2213 {
2214         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2215         portid_t  rxp;
2216         portid_t port_id;
2217         queueid_t rxq;
2218         uint16_t  nb_rx;
2219         uint16_t  i;
2220         uint8_t   j;
2221         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2222         uint64_t timer_period;
2223
2224         if (num_procs > 1) {
2225                 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2226                 return;
2227         }
2228
2229         /* convert to number of cycles */
2230         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2231
2232         for (j = 0; j < 2; j++) {
2233                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2234                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2235                                 port_id = fwd_ports_ids[rxp];
2236                                 /**
2237                                 * testpmd can stuck in the below do while loop
2238                                 * if rte_eth_rx_burst() always returns nonzero
2239                                 * packets. So timer is added to exit this loop
2240                                 * after 1sec timer expiry.
2241                                 */
2242                                 prev_tsc = rte_rdtsc();
2243                                 do {
2244                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2245                                                 pkts_burst, MAX_PKT_BURST);
2246                                         for (i = 0; i < nb_rx; i++)
2247                                                 rte_pktmbuf_free(pkts_burst[i]);
2248
2249                                         cur_tsc = rte_rdtsc();
2250                                         diff_tsc = cur_tsc - prev_tsc;
2251                                         timer_tsc += diff_tsc;
2252                                 } while ((nb_rx > 0) &&
2253                                         (timer_tsc < timer_period));
2254                                 timer_tsc = 0;
2255                         }
2256                 }
2257                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2258         }
2259 }
2260
2261 static void
2262 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2263 {
2264         struct fwd_stream **fsm;
2265         streamid_t nb_fs;
2266         streamid_t sm_id;
2267 #ifdef RTE_LIB_BITRATESTATS
2268         uint64_t tics_per_1sec;
2269         uint64_t tics_datum;
2270         uint64_t tics_current;
2271         uint16_t i, cnt_ports;
2272
2273         cnt_ports = nb_ports;
2274         tics_datum = rte_rdtsc();
2275         tics_per_1sec = rte_get_timer_hz();
2276 #endif
2277         fsm = &fwd_streams[fc->stream_idx];
2278         nb_fs = fc->stream_nb;
2279         do {
2280                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2281                         (*pkt_fwd)(fsm[sm_id]);
2282 #ifdef RTE_LIB_BITRATESTATS
2283                 if (bitrate_enabled != 0 &&
2284                                 bitrate_lcore_id == rte_lcore_id()) {
2285                         tics_current = rte_rdtsc();
2286                         if (tics_current - tics_datum >= tics_per_1sec) {
2287                                 /* Periodic bitrate calculation */
2288                                 for (i = 0; i < cnt_ports; i++)
2289                                         rte_stats_bitrate_calc(bitrate_data,
2290                                                 ports_ids[i]);
2291                                 tics_datum = tics_current;
2292                         }
2293                 }
2294 #endif
2295 #ifdef RTE_LIB_LATENCYSTATS
2296                 if (latencystats_enabled != 0 &&
2297                                 latencystats_lcore_id == rte_lcore_id())
2298                         rte_latencystats_update();
2299 #endif
2300
2301         } while (! fc->stopped);
2302 }
2303
2304 static int
2305 start_pkt_forward_on_core(void *fwd_arg)
2306 {
2307         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2308                              cur_fwd_config.fwd_eng->packet_fwd);
2309         return 0;
2310 }
2311
2312 /*
2313  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2314  * Used to start communication flows in network loopback test configurations.
2315  */
2316 static int
2317 run_one_txonly_burst_on_core(void *fwd_arg)
2318 {
2319         struct fwd_lcore *fwd_lc;
2320         struct fwd_lcore tmp_lcore;
2321
2322         fwd_lc = (struct fwd_lcore *) fwd_arg;
2323         tmp_lcore = *fwd_lc;
2324         tmp_lcore.stopped = 1;
2325         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2326         return 0;
2327 }
2328
2329 /*
2330  * Launch packet forwarding:
2331  *     - Setup per-port forwarding context.
2332  *     - launch logical cores with their forwarding configuration.
2333  */
2334 static void
2335 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2336 {
2337         unsigned int i;
2338         unsigned int lc_id;
2339         int diag;
2340
2341         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2342                 lc_id = fwd_lcores_cpuids[i];
2343                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2344                         fwd_lcores[i]->stopped = 0;
2345                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2346                                                      fwd_lcores[i], lc_id);
2347                         if (diag != 0)
2348                                 fprintf(stderr,
2349                                         "launch lcore %u failed - diag=%d\n",
2350                                         lc_id, diag);
2351                 }
2352         }
2353 }
2354
2355 /*
2356  * Launch packet forwarding configuration.
2357  */
2358 void
2359 start_packet_forwarding(int with_tx_first)
2360 {
2361         port_fwd_begin_t port_fwd_begin;
2362         port_fwd_end_t  port_fwd_end;
2363         unsigned int i;
2364
2365         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2366                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2367
2368         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2369                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2370
2371         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2372                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2373                 (!nb_rxq || !nb_txq))
2374                 rte_exit(EXIT_FAILURE,
2375                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2376                         cur_fwd_eng->fwd_mode_name);
2377
2378         if (all_ports_started() == 0) {
2379                 fprintf(stderr, "Not all ports were started\n");
2380                 return;
2381         }
2382         if (test_done == 0) {
2383                 fprintf(stderr, "Packet forwarding already started\n");
2384                 return;
2385         }
2386
2387         fwd_config_setup();
2388
2389         pkt_fwd_config_display(&cur_fwd_config);
2390         if (!pkt_fwd_shared_rxq_check())
2391                 return;
2392
2393         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2394         if (port_fwd_begin != NULL) {
2395                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2396                         if (port_fwd_begin(fwd_ports_ids[i])) {
2397                                 fprintf(stderr,
2398                                         "Packet forwarding is not ready\n");
2399                                 return;
2400                         }
2401                 }
2402         }
2403
2404         if (with_tx_first) {
2405                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2406                 if (port_fwd_begin != NULL) {
2407                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2408                                 if (port_fwd_begin(fwd_ports_ids[i])) {
2409                                         fprintf(stderr,
2410                                                 "Packet forwarding is not ready\n");
2411                                         return;
2412                                 }
2413                         }
2414                 }
2415         }
2416
2417         test_done = 0;
2418
2419         if(!no_flush_rx)
2420                 flush_fwd_rx_queues();
2421
2422         rxtx_config_display();
2423
2424         fwd_stats_reset();
2425         if (with_tx_first) {
2426                 while (with_tx_first--) {
2427                         launch_packet_forwarding(
2428                                         run_one_txonly_burst_on_core);
2429                         rte_eal_mp_wait_lcore();
2430                 }
2431                 port_fwd_end = tx_only_engine.port_fwd_end;
2432                 if (port_fwd_end != NULL) {
2433                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2434                                 (*port_fwd_end)(fwd_ports_ids[i]);
2435                 }
2436         }
2437         launch_packet_forwarding(start_pkt_forward_on_core);
2438 }
2439
2440 void
2441 stop_packet_forwarding(void)
2442 {
2443         port_fwd_end_t port_fwd_end;
2444         lcoreid_t lc_id;
2445         portid_t pt_id;
2446         int i;
2447
2448         if (test_done) {
2449                 fprintf(stderr, "Packet forwarding not started\n");
2450                 return;
2451         }
2452         printf("Telling cores to stop...");
2453         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2454                 fwd_lcores[lc_id]->stopped = 1;
2455         printf("\nWaiting for lcores to finish...\n");
2456         rte_eal_mp_wait_lcore();
2457         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2458         if (port_fwd_end != NULL) {
2459                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2460                         pt_id = fwd_ports_ids[i];
2461                         (*port_fwd_end)(pt_id);
2462                 }
2463         }
2464
2465         fwd_stats_display();
2466
2467         printf("\nDone.\n");
2468         test_done = 1;
2469 }
2470
2471 void
2472 dev_set_link_up(portid_t pid)
2473 {
2474         if (rte_eth_dev_set_link_up(pid) < 0)
2475                 fprintf(stderr, "\nSet link up fail.\n");
2476 }
2477
2478 void
2479 dev_set_link_down(portid_t pid)
2480 {
2481         if (rte_eth_dev_set_link_down(pid) < 0)
2482                 fprintf(stderr, "\nSet link down fail.\n");
2483 }
2484
2485 static int
2486 all_ports_started(void)
2487 {
2488         portid_t pi;
2489         struct rte_port *port;
2490
2491         RTE_ETH_FOREACH_DEV(pi) {
2492                 port = &ports[pi];
2493                 /* Check if there is a port which is not started */
2494                 if ((port->port_status != RTE_PORT_STARTED) &&
2495                         (port->slave_flag == 0))
2496                         return 0;
2497         }
2498
2499         /* No port is not started */
2500         return 1;
2501 }
2502
2503 int
2504 port_is_stopped(portid_t port_id)
2505 {
2506         struct rte_port *port = &ports[port_id];
2507
2508         if ((port->port_status != RTE_PORT_STOPPED) &&
2509             (port->slave_flag == 0))
2510                 return 0;
2511         return 1;
2512 }
2513
2514 int
2515 all_ports_stopped(void)
2516 {
2517         portid_t pi;
2518
2519         RTE_ETH_FOREACH_DEV(pi) {
2520                 if (!port_is_stopped(pi))
2521                         return 0;
2522         }
2523
2524         return 1;
2525 }
2526
2527 int
2528 port_is_started(portid_t port_id)
2529 {
2530         if (port_id_is_invalid(port_id, ENABLED_WARN))
2531                 return 0;
2532
2533         if (ports[port_id].port_status != RTE_PORT_STARTED)
2534                 return 0;
2535
2536         return 1;
2537 }
2538
2539 /* Configure the Rx and Tx hairpin queues for the selected port. */
2540 static int
2541 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2542 {
2543         queueid_t qi;
2544         struct rte_eth_hairpin_conf hairpin_conf = {
2545                 .peer_count = 1,
2546         };
2547         int i;
2548         int diag;
2549         struct rte_port *port = &ports[pi];
2550         uint16_t peer_rx_port = pi;
2551         uint16_t peer_tx_port = pi;
2552         uint32_t manual = 1;
2553         uint32_t tx_exp = hairpin_mode & 0x10;
2554
2555         if (!(hairpin_mode & 0xf)) {
2556                 peer_rx_port = pi;
2557                 peer_tx_port = pi;
2558                 manual = 0;
2559         } else if (hairpin_mode & 0x1) {
2560                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2561                                                        RTE_ETH_DEV_NO_OWNER);
2562                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2563                         peer_tx_port = rte_eth_find_next_owned_by(0,
2564                                                 RTE_ETH_DEV_NO_OWNER);
2565                 if (p_pi != RTE_MAX_ETHPORTS) {
2566                         peer_rx_port = p_pi;
2567                 } else {
2568                         uint16_t next_pi;
2569
2570                         /* Last port will be the peer RX port of the first. */
2571                         RTE_ETH_FOREACH_DEV(next_pi)
2572                                 peer_rx_port = next_pi;
2573                 }
2574                 manual = 1;
2575         } else if (hairpin_mode & 0x2) {
2576                 if (cnt_pi & 0x1) {
2577                         peer_rx_port = p_pi;
2578                 } else {
2579                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2580                                                 RTE_ETH_DEV_NO_OWNER);
2581                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2582                                 peer_rx_port = pi;
2583                 }
2584                 peer_tx_port = peer_rx_port;
2585                 manual = 1;
2586         }
2587
2588         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2589                 hairpin_conf.peers[0].port = peer_rx_port;
2590                 hairpin_conf.peers[0].queue = i + nb_rxq;
2591                 hairpin_conf.manual_bind = !!manual;
2592                 hairpin_conf.tx_explicit = !!tx_exp;
2593                 diag = rte_eth_tx_hairpin_queue_setup
2594                         (pi, qi, nb_txd, &hairpin_conf);
2595                 i++;
2596                 if (diag == 0)
2597                         continue;
2598
2599                 /* Fail to setup rx queue, return */
2600                 if (port->port_status == RTE_PORT_HANDLING)
2601                         port->port_status = RTE_PORT_STOPPED;
2602                 else
2603                         fprintf(stderr,
2604                                 "Port %d can not be set back to stopped\n", pi);
2605                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2606                         pi);
2607                 /* try to reconfigure queues next time */
2608                 port->need_reconfig_queues = 1;
2609                 return -1;
2610         }
2611         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2612                 hairpin_conf.peers[0].port = peer_tx_port;
2613                 hairpin_conf.peers[0].queue = i + nb_txq;
2614                 hairpin_conf.manual_bind = !!manual;
2615                 hairpin_conf.tx_explicit = !!tx_exp;
2616                 diag = rte_eth_rx_hairpin_queue_setup
2617                         (pi, qi, nb_rxd, &hairpin_conf);
2618                 i++;
2619                 if (diag == 0)
2620                         continue;
2621
2622                 /* Fail to setup rx queue, return */
2623                 if (port->port_status == RTE_PORT_HANDLING)
2624                         port->port_status = RTE_PORT_STOPPED;
2625                 else
2626                         fprintf(stderr,
2627                                 "Port %d can not be set back to stopped\n", pi);
2628                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2629                         pi);
2630                 /* try to reconfigure queues next time */
2631                 port->need_reconfig_queues = 1;
2632                 return -1;
2633         }
2634         return 0;
2635 }
2636
2637 /* Configure the Rx with optional split. */
2638 int
2639 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2640                uint16_t nb_rx_desc, unsigned int socket_id,
2641                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2642 {
2643         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2644         unsigned int i, mp_n;
2645         int ret;
2646
2647         if (rx_pkt_nb_segs <= 1 ||
2648             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2649                 rx_conf->rx_seg = NULL;
2650                 rx_conf->rx_nseg = 0;
2651                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2652                                              nb_rx_desc, socket_id,
2653                                              rx_conf, mp);
2654                 return ret;
2655         }
2656         for (i = 0; i < rx_pkt_nb_segs; i++) {
2657                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2658                 struct rte_mempool *mpx;
2659                 /*
2660                  * Use last valid pool for the segments with number
2661                  * exceeding the pool index.
2662                  */
2663                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2664                 mpx = mbuf_pool_find(socket_id, mp_n);
2665                 /* Handle zero as mbuf data buffer size. */
2666                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2667                                    rx_pkt_seg_lengths[i] :
2668                                    mbuf_data_size[mp_n];
2669                 rx_seg->offset = i < rx_pkt_nb_offs ?
2670                                    rx_pkt_seg_offsets[i] : 0;
2671                 rx_seg->mp = mpx ? mpx : mp;
2672         }
2673         rx_conf->rx_nseg = rx_pkt_nb_segs;
2674         rx_conf->rx_seg = rx_useg;
2675         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2676                                     socket_id, rx_conf, NULL);
2677         rx_conf->rx_seg = NULL;
2678         rx_conf->rx_nseg = 0;
2679         return ret;
2680 }
2681
2682 static int
2683 alloc_xstats_display_info(portid_t pi)
2684 {
2685         uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2686         uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2687         uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2688
2689         if (xstats_display_num == 0)
2690                 return 0;
2691
2692         *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2693         if (*ids_supp == NULL)
2694                 goto fail_ids_supp;
2695
2696         *prev_values = calloc(xstats_display_num,
2697                               sizeof(**prev_values));
2698         if (*prev_values == NULL)
2699                 goto fail_prev_values;
2700
2701         *curr_values = calloc(xstats_display_num,
2702                               sizeof(**curr_values));
2703         if (*curr_values == NULL)
2704                 goto fail_curr_values;
2705
2706         ports[pi].xstats_info.allocated = true;
2707
2708         return 0;
2709
2710 fail_curr_values:
2711         free(*prev_values);
2712 fail_prev_values:
2713         free(*ids_supp);
2714 fail_ids_supp:
2715         return -ENOMEM;
2716 }
2717
2718 static void
2719 free_xstats_display_info(portid_t pi)
2720 {
2721         if (!ports[pi].xstats_info.allocated)
2722                 return;
2723         free(ports[pi].xstats_info.ids_supp);
2724         free(ports[pi].xstats_info.prev_values);
2725         free(ports[pi].xstats_info.curr_values);
2726         ports[pi].xstats_info.allocated = false;
2727 }
2728
2729 /** Fill helper structures for specified port to show extended statistics. */
2730 static void
2731 fill_xstats_display_info_for_port(portid_t pi)
2732 {
2733         unsigned int stat, stat_supp;
2734         const char *xstat_name;
2735         struct rte_port *port;
2736         uint64_t *ids_supp;
2737         int rc;
2738
2739         if (xstats_display_num == 0)
2740                 return;
2741
2742         if (pi == (portid_t)RTE_PORT_ALL) {
2743                 fill_xstats_display_info();
2744                 return;
2745         }
2746
2747         port = &ports[pi];
2748         if (port->port_status != RTE_PORT_STARTED)
2749                 return;
2750
2751         if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2752                 rte_exit(EXIT_FAILURE,
2753                          "Failed to allocate xstats display memory\n");
2754
2755         ids_supp = port->xstats_info.ids_supp;
2756         for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2757                 xstat_name = xstats_display[stat].name;
2758                 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2759                                                    ids_supp + stat_supp);
2760                 if (rc != 0) {
2761                         fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2762                                 xstat_name, pi, stat);
2763                         continue;
2764                 }
2765                 stat_supp++;
2766         }
2767
2768         port->xstats_info.ids_supp_sz = stat_supp;
2769 }
2770
2771 /** Fill helper structures for all ports to show extended statistics. */
2772 static void
2773 fill_xstats_display_info(void)
2774 {
2775         portid_t pi;
2776
2777         if (xstats_display_num == 0)
2778                 return;
2779
2780         RTE_ETH_FOREACH_DEV(pi)
2781                 fill_xstats_display_info_for_port(pi);
2782 }
2783
2784 int
2785 start_port(portid_t pid)
2786 {
2787         int diag, need_check_link_status = -1;
2788         portid_t pi;
2789         portid_t p_pi = RTE_MAX_ETHPORTS;
2790         portid_t pl[RTE_MAX_ETHPORTS];
2791         portid_t peer_pl[RTE_MAX_ETHPORTS];
2792         uint16_t cnt_pi = 0;
2793         uint16_t cfg_pi = 0;
2794         int peer_pi;
2795         queueid_t qi;
2796         struct rte_port *port;
2797         struct rte_eth_hairpin_cap cap;
2798
2799         if (port_id_is_invalid(pid, ENABLED_WARN))
2800                 return 0;
2801
2802         RTE_ETH_FOREACH_DEV(pi) {
2803                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2804                         continue;
2805
2806                 if (port_is_bonding_slave(pi)) {
2807                         fprintf(stderr,
2808                                 "Please remove port %d from bonded device.\n",
2809                                 pi);
2810                         continue;
2811                 }
2812
2813                 need_check_link_status = 0;
2814                 port = &ports[pi];
2815                 if (port->port_status == RTE_PORT_STOPPED)
2816                         port->port_status = RTE_PORT_HANDLING;
2817                 else {
2818                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2819                         continue;
2820                 }
2821
2822                 if (port->need_reconfig > 0) {
2823                         struct rte_eth_conf dev_conf;
2824                         int k;
2825
2826                         port->need_reconfig = 0;
2827
2828                         if (flow_isolate_all) {
2829                                 int ret = port_flow_isolate(pi, 1);
2830                                 if (ret) {
2831                                         fprintf(stderr,
2832                                                 "Failed to apply isolated mode on port %d\n",
2833                                                 pi);
2834                                         return -1;
2835                                 }
2836                         }
2837                         configure_rxtx_dump_callbacks(0);
2838                         printf("Configuring Port %d (socket %u)\n", pi,
2839                                         port->socket_id);
2840                         if (nb_hairpinq > 0 &&
2841                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2842                                 fprintf(stderr,
2843                                         "Port %d doesn't support hairpin queues\n",
2844                                         pi);
2845                                 return -1;
2846                         }
2847
2848                         /* configure port */
2849                         diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2850                                                      nb_txq + nb_hairpinq,
2851                                                      &(port->dev_conf));
2852                         if (diag != 0) {
2853                                 if (port->port_status == RTE_PORT_HANDLING)
2854                                         port->port_status = RTE_PORT_STOPPED;
2855                                 else
2856                                         fprintf(stderr,
2857                                                 "Port %d can not be set back to stopped\n",
2858                                                 pi);
2859                                 fprintf(stderr, "Fail to configure port %d\n",
2860                                         pi);
2861                                 /* try to reconfigure port next time */
2862                                 port->need_reconfig = 1;
2863                                 return -1;
2864                         }
2865                         /* get device configuration*/
2866                         if (0 !=
2867                                 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2868                                 fprintf(stderr,
2869                                         "port %d can not get device configuration\n",
2870                                         pi);
2871                                 return -1;
2872                         }
2873                         /* Apply Rx offloads configuration */
2874                         if (dev_conf.rxmode.offloads !=
2875                             port->dev_conf.rxmode.offloads) {
2876                                 port->dev_conf.rxmode.offloads |=
2877                                         dev_conf.rxmode.offloads;
2878                                 for (k = 0;
2879                                      k < port->dev_info.max_rx_queues;
2880                                      k++)
2881                                         port->rx_conf[k].offloads |=
2882                                                 dev_conf.rxmode.offloads;
2883                         }
2884                         /* Apply Tx offloads configuration */
2885                         if (dev_conf.txmode.offloads !=
2886                             port->dev_conf.txmode.offloads) {
2887                                 port->dev_conf.txmode.offloads |=
2888                                         dev_conf.txmode.offloads;
2889                                 for (k = 0;
2890                                      k < port->dev_info.max_tx_queues;
2891                                      k++)
2892                                         port->tx_conf[k].offloads |=
2893                                                 dev_conf.txmode.offloads;
2894                         }
2895                 }
2896                 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2897                         port->need_reconfig_queues = 0;
2898                         /* setup tx queues */
2899                         for (qi = 0; qi < nb_txq; qi++) {
2900                                 if ((numa_support) &&
2901                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2902                                         diag = rte_eth_tx_queue_setup(pi, qi,
2903                                                 port->nb_tx_desc[qi],
2904                                                 txring_numa[pi],
2905                                                 &(port->tx_conf[qi]));
2906                                 else
2907                                         diag = rte_eth_tx_queue_setup(pi, qi,
2908                                                 port->nb_tx_desc[qi],
2909                                                 port->socket_id,
2910                                                 &(port->tx_conf[qi]));
2911
2912                                 if (diag == 0)
2913                                         continue;
2914
2915                                 /* Fail to setup tx queue, return */
2916                                 if (port->port_status == RTE_PORT_HANDLING)
2917                                         port->port_status = RTE_PORT_STOPPED;
2918                                 else
2919                                         fprintf(stderr,
2920                                                 "Port %d can not be set back to stopped\n",
2921                                                 pi);
2922                                 fprintf(stderr,
2923                                         "Fail to configure port %d tx queues\n",
2924                                         pi);
2925                                 /* try to reconfigure queues next time */
2926                                 port->need_reconfig_queues = 1;
2927                                 return -1;
2928                         }
2929                         for (qi = 0; qi < nb_rxq; qi++) {
2930                                 /* setup rx queues */
2931                                 if ((numa_support) &&
2932                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2933                                         struct rte_mempool * mp =
2934                                                 mbuf_pool_find
2935                                                         (rxring_numa[pi], 0);
2936                                         if (mp == NULL) {
2937                                                 fprintf(stderr,
2938                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2939                                                         rxring_numa[pi]);
2940                                                 return -1;
2941                                         }
2942
2943                                         diag = rx_queue_setup(pi, qi,
2944                                              port->nb_rx_desc[qi],
2945                                              rxring_numa[pi],
2946                                              &(port->rx_conf[qi]),
2947                                              mp);
2948                                 } else {
2949                                         struct rte_mempool *mp =
2950                                                 mbuf_pool_find
2951                                                         (port->socket_id, 0);
2952                                         if (mp == NULL) {
2953                                                 fprintf(stderr,
2954                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2955                                                         port->socket_id);
2956                                                 return -1;
2957                                         }
2958                                         diag = rx_queue_setup(pi, qi,
2959                                              port->nb_rx_desc[qi],
2960                                              port->socket_id,
2961                                              &(port->rx_conf[qi]),
2962                                              mp);
2963                                 }
2964                                 if (diag == 0)
2965                                         continue;
2966
2967                                 /* Fail to setup rx queue, return */
2968                                 if (port->port_status == RTE_PORT_HANDLING)
2969                                         port->port_status = RTE_PORT_STOPPED;
2970                                 else
2971                                         fprintf(stderr,
2972                                                 "Port %d can not be set back to stopped\n",
2973                                                 pi);
2974                                 fprintf(stderr,
2975                                         "Fail to configure port %d rx queues\n",
2976                                         pi);
2977                                 /* try to reconfigure queues next time */
2978                                 port->need_reconfig_queues = 1;
2979                                 return -1;
2980                         }
2981                         /* setup hairpin queues */
2982                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2983                                 return -1;
2984                 }
2985                 configure_rxtx_dump_callbacks(verbose_level);
2986                 if (clear_ptypes) {
2987                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2988                                         NULL, 0);
2989                         if (diag < 0)
2990                                 fprintf(stderr,
2991                                         "Port %d: Failed to disable Ptype parsing\n",
2992                                         pi);
2993                 }
2994
2995                 p_pi = pi;
2996                 cnt_pi++;
2997
2998                 /* start port */
2999                 diag = eth_dev_start_mp(pi);
3000                 if (diag < 0) {
3001                         fprintf(stderr, "Fail to start port %d: %s\n",
3002                                 pi, rte_strerror(-diag));
3003
3004                         /* Fail to setup rx queue, return */
3005                         if (port->port_status == RTE_PORT_HANDLING)
3006                                 port->port_status = RTE_PORT_STOPPED;
3007                         else
3008                                 fprintf(stderr,
3009                                         "Port %d can not be set back to stopped\n",
3010                                         pi);
3011                         continue;
3012                 }
3013
3014                 if (port->port_status == RTE_PORT_HANDLING)
3015                         port->port_status = RTE_PORT_STARTED;
3016                 else
3017                         fprintf(stderr, "Port %d can not be set into started\n",
3018                                 pi);
3019
3020                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
3021                         printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
3022                                         RTE_ETHER_ADDR_BYTES(&port->eth_addr));
3023
3024                 /* at least one port started, need checking link status */
3025                 need_check_link_status = 1;
3026
3027                 pl[cfg_pi++] = pi;
3028         }
3029
3030         if (need_check_link_status == 1 && !no_link_check)
3031                 check_all_ports_link_status(RTE_PORT_ALL);
3032         else if (need_check_link_status == 0)
3033                 fprintf(stderr, "Please stop the ports first\n");
3034
3035         if (hairpin_mode & 0xf) {
3036                 uint16_t i;
3037                 int j;
3038
3039                 /* bind all started hairpin ports */
3040                 for (i = 0; i < cfg_pi; i++) {
3041                         pi = pl[i];
3042                         /* bind current Tx to all peer Rx */
3043                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3044                                                         RTE_MAX_ETHPORTS, 1);
3045                         if (peer_pi < 0)
3046                                 return peer_pi;
3047                         for (j = 0; j < peer_pi; j++) {
3048                                 if (!port_is_started(peer_pl[j]))
3049                                         continue;
3050                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
3051                                 if (diag < 0) {
3052                                         fprintf(stderr,
3053                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3054                                                 pi, peer_pl[j],
3055                                                 rte_strerror(-diag));
3056                                         return -1;
3057                                 }
3058                         }
3059                         /* bind all peer Tx to current Rx */
3060                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3061                                                         RTE_MAX_ETHPORTS, 0);
3062                         if (peer_pi < 0)
3063                                 return peer_pi;
3064                         for (j = 0; j < peer_pi; j++) {
3065                                 if (!port_is_started(peer_pl[j]))
3066                                         continue;
3067                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
3068                                 if (diag < 0) {
3069                                         fprintf(stderr,
3070                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3071                                                 peer_pl[j], pi,
3072                                                 rte_strerror(-diag));
3073                                         return -1;
3074                                 }
3075                         }
3076                 }
3077         }
3078
3079         fill_xstats_display_info_for_port(pid);
3080
3081         printf("Done\n");
3082         return 0;
3083 }
3084
3085 void
3086 stop_port(portid_t pid)
3087 {
3088         portid_t pi;
3089         struct rte_port *port;
3090         int need_check_link_status = 0;
3091         portid_t peer_pl[RTE_MAX_ETHPORTS];
3092         int peer_pi;
3093
3094         if (port_id_is_invalid(pid, ENABLED_WARN))
3095                 return;
3096
3097         printf("Stopping ports...\n");
3098
3099         RTE_ETH_FOREACH_DEV(pi) {
3100                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3101                         continue;
3102
3103                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3104                         fprintf(stderr,
3105                                 "Please remove port %d from forwarding configuration.\n",
3106                                 pi);
3107                         continue;
3108                 }
3109
3110                 if (port_is_bonding_slave(pi)) {
3111                         fprintf(stderr,
3112                                 "Please remove port %d from bonded device.\n",
3113                                 pi);
3114                         continue;
3115                 }
3116
3117                 port = &ports[pi];
3118                 if (port->port_status == RTE_PORT_STARTED)
3119                         port->port_status = RTE_PORT_HANDLING;
3120                 else
3121                         continue;
3122
3123                 if (hairpin_mode & 0xf) {
3124                         int j;
3125
3126                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3127                         /* unbind all peer Tx from current Rx */
3128                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3129                                                         RTE_MAX_ETHPORTS, 0);
3130                         if (peer_pi < 0)
3131                                 continue;
3132                         for (j = 0; j < peer_pi; j++) {
3133                                 if (!port_is_started(peer_pl[j]))
3134                                         continue;
3135                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
3136                         }
3137                 }
3138
3139                 if (port->flow_list)
3140                         port_flow_flush(pi);
3141
3142                 if (eth_dev_stop_mp(pi) != 0)
3143                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3144                                 pi);
3145
3146                 if (port->port_status == RTE_PORT_HANDLING)
3147                         port->port_status = RTE_PORT_STOPPED;
3148                 else
3149                         fprintf(stderr, "Port %d can not be set into stopped\n",
3150                                 pi);
3151                 need_check_link_status = 1;
3152         }
3153         if (need_check_link_status && !no_link_check)
3154                 check_all_ports_link_status(RTE_PORT_ALL);
3155
3156         printf("Done\n");
3157 }
3158
3159 static void
3160 remove_invalid_ports_in(portid_t *array, portid_t *total)
3161 {
3162         portid_t i;
3163         portid_t new_total = 0;
3164
3165         for (i = 0; i < *total; i++)
3166                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3167                         array[new_total] = array[i];
3168                         new_total++;
3169                 }
3170         *total = new_total;
3171 }
3172
3173 static void
3174 remove_invalid_ports(void)
3175 {
3176         remove_invalid_ports_in(ports_ids, &nb_ports);
3177         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3178         nb_cfg_ports = nb_fwd_ports;
3179 }
3180
3181 void
3182 close_port(portid_t pid)
3183 {
3184         portid_t pi;
3185         struct rte_port *port;
3186
3187         if (port_id_is_invalid(pid, ENABLED_WARN))
3188                 return;
3189
3190         printf("Closing ports...\n");
3191
3192         RTE_ETH_FOREACH_DEV(pi) {
3193                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3194                         continue;
3195
3196                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3197                         fprintf(stderr,
3198                                 "Please remove port %d from forwarding configuration.\n",
3199                                 pi);
3200                         continue;
3201                 }
3202
3203                 if (port_is_bonding_slave(pi)) {
3204                         fprintf(stderr,
3205                                 "Please remove port %d from bonded device.\n",
3206                                 pi);
3207                         continue;
3208                 }
3209
3210                 port = &ports[pi];
3211                 if (port->port_status == RTE_PORT_CLOSED) {
3212                         fprintf(stderr, "Port %d is already closed\n", pi);
3213                         continue;
3214                 }
3215
3216                 if (is_proc_primary()) {
3217                         port_flow_flush(pi);
3218                         port_flex_item_flush(pi);
3219                         port_action_handle_flush(pi);
3220                         rte_eth_dev_close(pi);
3221                 }
3222
3223                 free_xstats_display_info(pi);
3224         }
3225
3226         remove_invalid_ports();
3227         printf("Done\n");
3228 }
3229
3230 void
3231 reset_port(portid_t pid)
3232 {
3233         int diag;
3234         portid_t pi;
3235         struct rte_port *port;
3236
3237         if (port_id_is_invalid(pid, ENABLED_WARN))
3238                 return;
3239
3240         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3241                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3242                 fprintf(stderr,
3243                         "Can not reset port(s), please stop port(s) first.\n");
3244                 return;
3245         }
3246
3247         printf("Resetting ports...\n");
3248
3249         RTE_ETH_FOREACH_DEV(pi) {
3250                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3251                         continue;
3252
3253                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3254                         fprintf(stderr,
3255                                 "Please remove port %d from forwarding configuration.\n",
3256                                 pi);
3257                         continue;
3258                 }
3259
3260                 if (port_is_bonding_slave(pi)) {
3261                         fprintf(stderr,
3262                                 "Please remove port %d from bonded device.\n",
3263                                 pi);
3264                         continue;
3265                 }
3266
3267                 diag = rte_eth_dev_reset(pi);
3268                 if (diag == 0) {
3269                         port = &ports[pi];
3270                         port->need_reconfig = 1;
3271                         port->need_reconfig_queues = 1;
3272                 } else {
3273                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3274                                 pi, diag);
3275                 }
3276         }
3277
3278         printf("Done\n");
3279 }
3280
3281 void
3282 attach_port(char *identifier)
3283 {
3284         portid_t pi;
3285         struct rte_dev_iterator iterator;
3286
3287         printf("Attaching a new port...\n");
3288
3289         if (identifier == NULL) {
3290                 fprintf(stderr, "Invalid parameters are specified\n");
3291                 return;
3292         }
3293
3294         if (rte_dev_probe(identifier) < 0) {
3295                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3296                 return;
3297         }
3298
3299         /* first attach mode: event */
3300         if (setup_on_probe_event) {
3301                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3302                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3303                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
3304                                         ports[pi].need_setup != 0)
3305                                 setup_attached_port(pi);
3306                 return;
3307         }
3308
3309         /* second attach mode: iterator */
3310         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3311                 /* setup ports matching the devargs used for probing */
3312                 if (port_is_forwarding(pi))
3313                         continue; /* port was already attached before */
3314                 setup_attached_port(pi);
3315         }
3316 }
3317
3318 static void
3319 setup_attached_port(portid_t pi)
3320 {
3321         unsigned int socket_id;
3322         int ret;
3323
3324         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3325         /* if socket_id is invalid, set to the first available socket. */
3326         if (check_socket_id(socket_id) < 0)
3327                 socket_id = socket_ids[0];
3328         reconfig(pi, socket_id);
3329         ret = rte_eth_promiscuous_enable(pi);
3330         if (ret != 0)
3331                 fprintf(stderr,
3332                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3333                         pi, rte_strerror(-ret));
3334
3335         ports_ids[nb_ports++] = pi;
3336         fwd_ports_ids[nb_fwd_ports++] = pi;
3337         nb_cfg_ports = nb_fwd_ports;
3338         ports[pi].need_setup = 0;
3339         ports[pi].port_status = RTE_PORT_STOPPED;
3340
3341         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3342         printf("Done\n");
3343 }
3344
3345 static void
3346 detach_device(struct rte_device *dev)
3347 {
3348         portid_t sibling;
3349
3350         if (dev == NULL) {
3351                 fprintf(stderr, "Device already removed\n");
3352                 return;
3353         }
3354
3355         printf("Removing a device...\n");
3356
3357         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3358                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3359                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3360                                 fprintf(stderr, "Port %u not stopped\n",
3361                                         sibling);
3362                                 return;
3363                         }
3364                         port_flow_flush(sibling);
3365                 }
3366         }
3367
3368         if (rte_dev_remove(dev) < 0) {
3369                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3370                 return;
3371         }
3372         remove_invalid_ports();
3373
3374         printf("Device is detached\n");
3375         printf("Now total ports is %d\n", nb_ports);
3376         printf("Done\n");
3377         return;
3378 }
3379
3380 void
3381 detach_port_device(portid_t port_id)
3382 {
3383         int ret;
3384         struct rte_eth_dev_info dev_info;
3385
3386         if (port_id_is_invalid(port_id, ENABLED_WARN))
3387                 return;
3388
3389         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3390                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3391                         fprintf(stderr, "Port not stopped\n");
3392                         return;
3393                 }
3394                 fprintf(stderr, "Port was not closed\n");
3395         }
3396
3397         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3398         if (ret != 0) {
3399                 TESTPMD_LOG(ERR,
3400                         "Failed to get device info for port %d, not detaching\n",
3401                         port_id);
3402                 return;
3403         }
3404         detach_device(dev_info.device);
3405 }
3406
3407 void
3408 detach_devargs(char *identifier)
3409 {
3410         struct rte_dev_iterator iterator;
3411         struct rte_devargs da;
3412         portid_t port_id;
3413
3414         printf("Removing a device...\n");
3415
3416         memset(&da, 0, sizeof(da));
3417         if (rte_devargs_parsef(&da, "%s", identifier)) {
3418                 fprintf(stderr, "cannot parse identifier\n");
3419                 return;
3420         }
3421
3422         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3423                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3424                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3425                                 fprintf(stderr, "Port %u not stopped\n",
3426                                         port_id);
3427                                 rte_eth_iterator_cleanup(&iterator);
3428                                 rte_devargs_reset(&da);
3429                                 return;
3430                         }
3431                         port_flow_flush(port_id);
3432                 }
3433         }
3434
3435         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3436                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3437                             da.name, da.bus->name);
3438                 rte_devargs_reset(&da);
3439                 return;
3440         }
3441
3442         remove_invalid_ports();
3443
3444         printf("Device %s is detached\n", identifier);
3445         printf("Now total ports is %d\n", nb_ports);
3446         printf("Done\n");
3447         rte_devargs_reset(&da);
3448 }
3449
3450 void
3451 pmd_test_exit(void)
3452 {
3453         portid_t pt_id;
3454         unsigned int i;
3455         int ret;
3456
3457         if (test_done == 0)
3458                 stop_packet_forwarding();
3459
3460 #ifndef RTE_EXEC_ENV_WINDOWS
3461         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3462                 if (mempools[i]) {
3463                         if (mp_alloc_type == MP_ALLOC_ANON)
3464                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3465                                                      NULL);
3466                 }
3467         }
3468 #endif
3469         if (ports != NULL) {
3470                 no_link_check = 1;
3471                 RTE_ETH_FOREACH_DEV(pt_id) {
3472                         printf("\nStopping port %d...\n", pt_id);
3473                         fflush(stdout);
3474                         stop_port(pt_id);
3475                 }
3476                 RTE_ETH_FOREACH_DEV(pt_id) {
3477                         printf("\nShutting down port %d...\n", pt_id);
3478                         fflush(stdout);
3479                         close_port(pt_id);
3480                 }
3481         }
3482
3483         if (hot_plug) {
3484                 ret = rte_dev_event_monitor_stop();
3485                 if (ret) {
3486                         RTE_LOG(ERR, EAL,
3487                                 "fail to stop device event monitor.");
3488                         return;
3489                 }
3490
3491                 ret = rte_dev_event_callback_unregister(NULL,
3492                         dev_event_callback, NULL);
3493                 if (ret < 0) {
3494                         RTE_LOG(ERR, EAL,
3495                                 "fail to unregister device event callback.\n");
3496                         return;
3497                 }
3498
3499                 ret = rte_dev_hotplug_handle_disable();
3500                 if (ret) {
3501                         RTE_LOG(ERR, EAL,
3502                                 "fail to disable hotplug handling.\n");
3503                         return;
3504                 }
3505         }
3506         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3507                 if (mempools[i])
3508                         mempool_free_mp(mempools[i]);
3509         }
3510         free(xstats_display);
3511
3512         printf("\nBye...\n");
3513 }
3514
3515 typedef void (*cmd_func_t)(void);
3516 struct pmd_test_command {
3517         const char *cmd_name;
3518         cmd_func_t cmd_func;
3519 };
3520
3521 /* Check the link status of all ports in up to 9s, and print them finally */
3522 static void
3523 check_all_ports_link_status(uint32_t port_mask)
3524 {
3525 #define CHECK_INTERVAL 100 /* 100ms */
3526 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3527         portid_t portid;
3528         uint8_t count, all_ports_up, print_flag = 0;
3529         struct rte_eth_link link;
3530         int ret;
3531         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3532
3533         printf("Checking link statuses...\n");
3534         fflush(stdout);
3535         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3536                 all_ports_up = 1;
3537                 RTE_ETH_FOREACH_DEV(portid) {
3538                         if ((port_mask & (1 << portid)) == 0)
3539                                 continue;
3540                         memset(&link, 0, sizeof(link));
3541                         ret = rte_eth_link_get_nowait(portid, &link);
3542                         if (ret < 0) {
3543                                 all_ports_up = 0;
3544                                 if (print_flag == 1)
3545                                         fprintf(stderr,
3546                                                 "Port %u link get failed: %s\n",
3547                                                 portid, rte_strerror(-ret));
3548                                 continue;
3549                         }
3550                         /* print link status if flag set */
3551                         if (print_flag == 1) {
3552                                 rte_eth_link_to_str(link_status,
3553                                         sizeof(link_status), &link);
3554                                 printf("Port %d %s\n", portid, link_status);
3555                                 continue;
3556                         }
3557                         /* clear all_ports_up flag if any link down */
3558                         if (link.link_status == RTE_ETH_LINK_DOWN) {
3559                                 all_ports_up = 0;
3560                                 break;
3561                         }
3562                 }
3563                 /* after finally printing all link status, get out */
3564                 if (print_flag == 1)
3565                         break;
3566
3567                 if (all_ports_up == 0) {
3568                         fflush(stdout);
3569                         rte_delay_ms(CHECK_INTERVAL);
3570                 }
3571
3572                 /* set the print_flag if all ports up or timeout */
3573                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3574                         print_flag = 1;
3575                 }
3576
3577                 if (lsc_interrupt)
3578                         break;
3579         }
3580 }
3581
3582 static void
3583 rmv_port_callback(void *arg)
3584 {
3585         int need_to_start = 0;
3586         int org_no_link_check = no_link_check;
3587         portid_t port_id = (intptr_t)arg;
3588         struct rte_eth_dev_info dev_info;
3589         int ret;
3590
3591         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3592
3593         if (!test_done && port_is_forwarding(port_id)) {
3594                 need_to_start = 1;
3595                 stop_packet_forwarding();
3596         }
3597         no_link_check = 1;
3598         stop_port(port_id);
3599         no_link_check = org_no_link_check;
3600
3601         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3602         if (ret != 0)
3603                 TESTPMD_LOG(ERR,
3604                         "Failed to get device info for port %d, not detaching\n",
3605                         port_id);
3606         else {
3607                 struct rte_device *device = dev_info.device;
3608                 close_port(port_id);
3609                 detach_device(device); /* might be already removed or have more ports */
3610         }
3611         if (need_to_start)
3612                 start_packet_forwarding(0);
3613 }
3614
3615 /* This function is used by the interrupt thread */
3616 static int
3617 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3618                   void *ret_param)
3619 {
3620         RTE_SET_USED(param);
3621         RTE_SET_USED(ret_param);
3622
3623         if (type >= RTE_ETH_EVENT_MAX) {
3624                 fprintf(stderr,
3625                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3626                         port_id, __func__, type);
3627                 fflush(stderr);
3628         } else if (event_print_mask & (UINT32_C(1) << type)) {
3629                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3630                         eth_event_desc[type]);
3631                 fflush(stdout);
3632         }
3633
3634         switch (type) {
3635         case RTE_ETH_EVENT_NEW:
3636                 ports[port_id].need_setup = 1;
3637                 ports[port_id].port_status = RTE_PORT_HANDLING;
3638                 break;
3639         case RTE_ETH_EVENT_INTR_RMV:
3640                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3641                         break;
3642                 if (rte_eal_alarm_set(100000,
3643                                 rmv_port_callback, (void *)(intptr_t)port_id))
3644                         fprintf(stderr,
3645                                 "Could not set up deferred device removal\n");
3646                 break;
3647         case RTE_ETH_EVENT_DESTROY:
3648                 ports[port_id].port_status = RTE_PORT_CLOSED;
3649                 printf("Port %u is closed\n", port_id);
3650                 break;
3651         default:
3652                 break;
3653         }
3654         return 0;
3655 }
3656
3657 static int
3658 register_eth_event_callback(void)
3659 {
3660         int ret;
3661         enum rte_eth_event_type event;
3662
3663         for (event = RTE_ETH_EVENT_UNKNOWN;
3664                         event < RTE_ETH_EVENT_MAX; event++) {
3665                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3666                                 event,
3667                                 eth_event_callback,
3668                                 NULL);
3669                 if (ret != 0) {
3670                         TESTPMD_LOG(ERR, "Failed to register callback for "
3671                                         "%s event\n", eth_event_desc[event]);
3672                         return -1;
3673                 }
3674         }
3675
3676         return 0;
3677 }
3678
3679 /* This function is used by the interrupt thread */
3680 static void
3681 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3682                              __rte_unused void *arg)
3683 {
3684         uint16_t port_id;
3685         int ret;
3686
3687         if (type >= RTE_DEV_EVENT_MAX) {
3688                 fprintf(stderr, "%s called upon invalid event %d\n",
3689                         __func__, type);
3690                 fflush(stderr);
3691         }
3692
3693         switch (type) {
3694         case RTE_DEV_EVENT_REMOVE:
3695                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3696                         device_name);
3697                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3698                 if (ret) {
3699                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3700                                 device_name);
3701                         return;
3702                 }
3703                 /*
3704                  * Because the user's callback is invoked in eal interrupt
3705                  * callback, the interrupt callback need to be finished before
3706                  * it can be unregistered when detaching device. So finish
3707                  * callback soon and use a deferred removal to detach device
3708                  * is need. It is a workaround, once the device detaching be
3709                  * moved into the eal in the future, the deferred removal could
3710                  * be deleted.
3711                  */
3712                 if (rte_eal_alarm_set(100000,
3713                                 rmv_port_callback, (void *)(intptr_t)port_id))
3714                         RTE_LOG(ERR, EAL,
3715                                 "Could not set up deferred device removal\n");
3716                 break;
3717         case RTE_DEV_EVENT_ADD:
3718                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3719                         device_name);
3720                 /* TODO: After finish kernel driver binding,
3721                  * begin to attach port.
3722                  */
3723                 break;
3724         default:
3725                 break;
3726         }
3727 }
3728
3729 static void
3730 rxtx_port_config(portid_t pid)
3731 {
3732         uint16_t qid;
3733         uint64_t offloads;
3734         struct rte_port *port = &ports[pid];
3735
3736         for (qid = 0; qid < nb_rxq; qid++) {
3737                 offloads = port->rx_conf[qid].offloads;
3738                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3739
3740                 if (rxq_share > 0 &&
3741                     (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3742                         /* Non-zero share group to enable RxQ share. */
3743                         port->rx_conf[qid].share_group = pid / rxq_share + 1;
3744                         port->rx_conf[qid].share_qid = qid; /* Equal mapping. */
3745                 }
3746
3747                 if (offloads != 0)
3748                         port->rx_conf[qid].offloads = offloads;
3749
3750                 /* Check if any Rx parameters have been passed */
3751                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3752                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3753
3754                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3755                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3756
3757                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3758                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3759
3760                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3761                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3762
3763                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3764                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3765
3766                 port->nb_rx_desc[qid] = nb_rxd;
3767         }
3768
3769         for (qid = 0; qid < nb_txq; qid++) {
3770                 offloads = port->tx_conf[qid].offloads;
3771                 port->tx_conf[qid] = port->dev_info.default_txconf;
3772                 if (offloads != 0)
3773                         port->tx_conf[qid].offloads = offloads;
3774
3775                 /* Check if any Tx parameters have been passed */
3776                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3777                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3778
3779                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3780                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3781
3782                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3783                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3784
3785                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3786                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3787
3788                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3789                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3790
3791                 port->nb_tx_desc[qid] = nb_txd;
3792         }
3793 }
3794
3795 /*
3796  * Helper function to set MTU from frame size
3797  *
3798  * port->dev_info should be set before calling this function.
3799  *
3800  * return 0 on success, negative on error
3801  */
3802 int
3803 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3804 {
3805         struct rte_port *port = &ports[portid];
3806         uint32_t eth_overhead;
3807         uint16_t mtu, new_mtu;
3808
3809         eth_overhead = get_eth_overhead(&port->dev_info);
3810
3811         if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3812                 printf("Failed to get MTU for port %u\n", portid);
3813                 return -1;
3814         }
3815
3816         new_mtu = max_rx_pktlen - eth_overhead;
3817
3818         if (mtu == new_mtu)
3819                 return 0;
3820
3821         if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3822                 fprintf(stderr,
3823                         "Failed to set MTU to %u for port %u\n",
3824                         new_mtu, portid);
3825                 return -1;
3826         }
3827
3828         port->dev_conf.rxmode.mtu = new_mtu;
3829
3830         return 0;
3831 }
3832
3833 void
3834 init_port_config(void)
3835 {
3836         portid_t pid;
3837         struct rte_port *port;
3838         int ret, i;
3839
3840         RTE_ETH_FOREACH_DEV(pid) {
3841                 port = &ports[pid];
3842                 port->dev_conf.fdir_conf = fdir_conf;
3843
3844                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3845                 if (ret != 0)
3846                         return;
3847
3848                 if (nb_rxq > 1) {
3849                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3850                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3851                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3852                 } else {
3853                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3854                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3855                 }
3856
3857                 if (port->dcb_flag == 0) {
3858                         if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3859                                 port->dev_conf.rxmode.mq_mode =
3860                                         (enum rte_eth_rx_mq_mode)
3861                                                 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3862                         } else {
3863                                 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3864                                 port->dev_conf.rxmode.offloads &=
3865                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3866
3867                                 for (i = 0;
3868                                      i < port->dev_info.nb_rx_queues;
3869                                      i++)
3870                                         port->rx_conf[i].offloads &=
3871                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3872                         }
3873                 }
3874
3875                 rxtx_port_config(pid);
3876
3877                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3878                 if (ret != 0)
3879                         return;
3880
3881 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3882                 rte_pmd_ixgbe_bypass_init(pid);
3883 #endif
3884
3885                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3886                         port->dev_conf.intr_conf.lsc = 1;
3887                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3888                         port->dev_conf.intr_conf.rmv = 1;
3889         }
3890 }
3891
3892 void set_port_slave_flag(portid_t slave_pid)
3893 {
3894         struct rte_port *port;
3895
3896         port = &ports[slave_pid];
3897         port->slave_flag = 1;
3898 }
3899
3900 void clear_port_slave_flag(portid_t slave_pid)
3901 {
3902         struct rte_port *port;
3903
3904         port = &ports[slave_pid];
3905         port->slave_flag = 0;
3906 }
3907
3908 uint8_t port_is_bonding_slave(portid_t slave_pid)
3909 {
3910         struct rte_port *port;
3911         struct rte_eth_dev_info dev_info;
3912         int ret;
3913
3914         port = &ports[slave_pid];
3915         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3916         if (ret != 0) {
3917                 TESTPMD_LOG(ERR,
3918                         "Failed to get device info for port id %d,"
3919                         "cannot determine if the port is a bonded slave",
3920                         slave_pid);
3921                 return 0;
3922         }
3923         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3924                 return 1;
3925         return 0;
3926 }
3927
3928 const uint16_t vlan_tags[] = {
3929                 0,  1,  2,  3,  4,  5,  6,  7,
3930                 8,  9, 10, 11,  12, 13, 14, 15,
3931                 16, 17, 18, 19, 20, 21, 22, 23,
3932                 24, 25, 26, 27, 28, 29, 30, 31
3933 };
3934
3935 static  int
3936 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3937                  enum dcb_mode_enable dcb_mode,
3938                  enum rte_eth_nb_tcs num_tcs,
3939                  uint8_t pfc_en)
3940 {
3941         uint8_t i;
3942         int32_t rc;
3943         struct rte_eth_rss_conf rss_conf;
3944
3945         /*
3946          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3947          * given above, and the number of traffic classes available for use.
3948          */
3949         if (dcb_mode == DCB_VT_ENABLED) {
3950                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3951                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3952                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3953                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3954
3955                 /* VMDQ+DCB RX and TX configurations */
3956                 vmdq_rx_conf->enable_default_pool = 0;
3957                 vmdq_rx_conf->default_pool = 0;
3958                 vmdq_rx_conf->nb_queue_pools =
3959                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3960                 vmdq_tx_conf->nb_queue_pools =
3961                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3962
3963                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3964                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3965                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3966                         vmdq_rx_conf->pool_map[i].pools =
3967                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3968                 }
3969                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3970                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3971                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3972                 }
3973
3974                 /* set DCB mode of RX and TX of multiple queues */
3975                 eth_conf->rxmode.mq_mode =
3976                                 (enum rte_eth_rx_mq_mode)
3977                                         (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
3978                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
3979         } else {
3980                 struct rte_eth_dcb_rx_conf *rx_conf =
3981                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3982                 struct rte_eth_dcb_tx_conf *tx_conf =
3983                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3984
3985                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3986
3987                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3988                 if (rc != 0)
3989                         return rc;
3990
3991                 rx_conf->nb_tcs = num_tcs;
3992                 tx_conf->nb_tcs = num_tcs;
3993
3994                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3995                         rx_conf->dcb_tc[i] = i % num_tcs;
3996                         tx_conf->dcb_tc[i] = i % num_tcs;
3997                 }
3998
3999                 eth_conf->rxmode.mq_mode =
4000                                 (enum rte_eth_rx_mq_mode)
4001                                         (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
4002                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
4003                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
4004         }
4005
4006         if (pfc_en)
4007                 eth_conf->dcb_capability_en =
4008                                 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
4009         else
4010                 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
4011
4012         return 0;
4013 }
4014
4015 int
4016 init_port_dcb_config(portid_t pid,
4017                      enum dcb_mode_enable dcb_mode,
4018                      enum rte_eth_nb_tcs num_tcs,
4019                      uint8_t pfc_en)
4020 {
4021         struct rte_eth_conf port_conf;
4022         struct rte_port *rte_port;
4023         int retval;
4024         uint16_t i;
4025
4026         if (num_procs > 1) {
4027                 printf("The multi-process feature doesn't support dcb.\n");
4028                 return -ENOTSUP;
4029         }
4030         rte_port = &ports[pid];
4031
4032         /* retain the original device configuration. */
4033         memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
4034
4035         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
4036         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
4037         if (retval < 0)
4038                 return retval;
4039         port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4040         /* remove RSS HASH offload for DCB in vt mode */
4041         if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
4042                 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4043                 for (i = 0; i < nb_rxq; i++)
4044                         rte_port->rx_conf[i].offloads &=
4045                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4046         }
4047
4048         /* re-configure the device . */
4049         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
4050         if (retval < 0)
4051                 return retval;
4052
4053         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
4054         if (retval != 0)
4055                 return retval;
4056
4057         /* If dev_info.vmdq_pool_base is greater than 0,
4058          * the queue id of vmdq pools is started after pf queues.
4059          */
4060         if (dcb_mode == DCB_VT_ENABLED &&
4061             rte_port->dev_info.vmdq_pool_base > 0) {
4062                 fprintf(stderr,
4063                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
4064                         pid);
4065                 return -1;
4066         }
4067
4068         /* Assume the ports in testpmd have the same dcb capability
4069          * and has the same number of rxq and txq in dcb mode
4070          */
4071         if (dcb_mode == DCB_VT_ENABLED) {
4072                 if (rte_port->dev_info.max_vfs > 0) {
4073                         nb_rxq = rte_port->dev_info.nb_rx_queues;
4074                         nb_txq = rte_port->dev_info.nb_tx_queues;
4075                 } else {
4076                         nb_rxq = rte_port->dev_info.max_rx_queues;
4077                         nb_txq = rte_port->dev_info.max_tx_queues;
4078                 }
4079         } else {
4080                 /*if vt is disabled, use all pf queues */
4081                 if (rte_port->dev_info.vmdq_pool_base == 0) {
4082                         nb_rxq = rte_port->dev_info.max_rx_queues;
4083                         nb_txq = rte_port->dev_info.max_tx_queues;
4084                 } else {
4085                         nb_rxq = (queueid_t)num_tcs;
4086                         nb_txq = (queueid_t)num_tcs;
4087
4088                 }
4089         }
4090         rx_free_thresh = 64;
4091
4092         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4093
4094         rxtx_port_config(pid);
4095         /* VLAN filter */
4096         rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4097         for (i = 0; i < RTE_DIM(vlan_tags); i++)
4098                 rx_vft_set(pid, vlan_tags[i], 1);
4099
4100         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4101         if (retval != 0)
4102                 return retval;
4103
4104         rte_port->dcb_flag = 1;
4105
4106         /* Enter DCB configuration status */
4107         dcb_config = 1;
4108
4109         return 0;
4110 }
4111
4112 static void
4113 init_port(void)
4114 {
4115         int i;
4116
4117         /* Configuration of Ethernet ports. */
4118         ports = rte_zmalloc("testpmd: ports",
4119                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4120                             RTE_CACHE_LINE_SIZE);
4121         if (ports == NULL) {
4122                 rte_exit(EXIT_FAILURE,
4123                                 "rte_zmalloc(%d struct rte_port) failed\n",
4124                                 RTE_MAX_ETHPORTS);
4125         }
4126         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4127                 ports[i].xstats_info.allocated = false;
4128         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4129                 LIST_INIT(&ports[i].flow_tunnel_list);
4130         /* Initialize ports NUMA structures */
4131         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4132         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4133         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4134 }
4135
4136 static void
4137 force_quit(void)
4138 {
4139         pmd_test_exit();
4140         prompt_exit();
4141 }
4142
4143 static void
4144 print_stats(void)
4145 {
4146         uint8_t i;
4147         const char clr[] = { 27, '[', '2', 'J', '\0' };
4148         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4149
4150         /* Clear screen and move to top left */
4151         printf("%s%s", clr, top_left);
4152
4153         printf("\nPort statistics ====================================");
4154         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4155                 nic_stats_display(fwd_ports_ids[i]);
4156
4157         fflush(stdout);
4158 }
4159
4160 static void
4161 signal_handler(int signum)
4162 {
4163         if (signum == SIGINT || signum == SIGTERM) {
4164                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4165                         signum);
4166 #ifdef RTE_LIB_PDUMP
4167                 /* uninitialize packet capture framework */
4168                 rte_pdump_uninit();
4169 #endif
4170 #ifdef RTE_LIB_LATENCYSTATS
4171                 if (latencystats_enabled != 0)
4172                         rte_latencystats_uninit();
4173 #endif
4174                 force_quit();
4175                 /* Set flag to indicate the force termination. */
4176                 f_quit = 1;
4177                 /* exit with the expected status */
4178 #ifndef RTE_EXEC_ENV_WINDOWS
4179                 signal(signum, SIG_DFL);
4180                 kill(getpid(), signum);
4181 #endif
4182         }
4183 }
4184
4185 int
4186 main(int argc, char** argv)
4187 {
4188         int diag;
4189         portid_t port_id;
4190         uint16_t count;
4191         int ret;
4192
4193         signal(SIGINT, signal_handler);
4194         signal(SIGTERM, signal_handler);
4195
4196         testpmd_logtype = rte_log_register("testpmd");
4197         if (testpmd_logtype < 0)
4198                 rte_exit(EXIT_FAILURE, "Cannot register log type");
4199         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4200
4201         diag = rte_eal_init(argc, argv);
4202         if (diag < 0)
4203                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4204                          rte_strerror(rte_errno));
4205
4206         ret = register_eth_event_callback();
4207         if (ret != 0)
4208                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4209
4210 #ifdef RTE_LIB_PDUMP
4211         /* initialize packet capture framework */
4212         rte_pdump_init();
4213 #endif
4214
4215         count = 0;
4216         RTE_ETH_FOREACH_DEV(port_id) {
4217                 ports_ids[count] = port_id;
4218                 count++;
4219         }
4220         nb_ports = (portid_t) count;
4221         if (nb_ports == 0)
4222                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4223
4224         /* allocate port structures, and init them */
4225         init_port();
4226
4227         set_def_fwd_config();
4228         if (nb_lcores == 0)
4229                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4230                          "Check the core mask argument\n");
4231
4232         /* Bitrate/latency stats disabled by default */
4233 #ifdef RTE_LIB_BITRATESTATS
4234         bitrate_enabled = 0;
4235 #endif
4236 #ifdef RTE_LIB_LATENCYSTATS
4237         latencystats_enabled = 0;
4238 #endif
4239
4240         /* on FreeBSD, mlockall() is disabled by default */
4241 #ifdef RTE_EXEC_ENV_FREEBSD
4242         do_mlockall = 0;
4243 #else
4244         do_mlockall = 1;
4245 #endif
4246
4247         argc -= diag;
4248         argv += diag;
4249         if (argc > 1)
4250                 launch_args_parse(argc, argv);
4251
4252 #ifndef RTE_EXEC_ENV_WINDOWS
4253         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4254                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4255                         strerror(errno));
4256         }
4257 #endif
4258
4259         if (tx_first && interactive)
4260                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4261                                 "interactive mode.\n");
4262
4263         if (tx_first && lsc_interrupt) {
4264                 fprintf(stderr,
4265                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4266                 lsc_interrupt = 0;
4267         }
4268
4269         if (!nb_rxq && !nb_txq)
4270                 fprintf(stderr,
4271                         "Warning: Either rx or tx queues should be non-zero\n");
4272
4273         if (nb_rxq > 1 && nb_rxq > nb_txq)
4274                 fprintf(stderr,
4275                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4276                         nb_rxq, nb_txq);
4277
4278         init_config();
4279
4280         if (hot_plug) {
4281                 ret = rte_dev_hotplug_handle_enable();
4282                 if (ret) {
4283                         RTE_LOG(ERR, EAL,
4284                                 "fail to enable hotplug handling.");
4285                         return -1;
4286                 }
4287
4288                 ret = rte_dev_event_monitor_start();
4289                 if (ret) {
4290                         RTE_LOG(ERR, EAL,
4291                                 "fail to start device event monitoring.");
4292                         return -1;
4293                 }
4294
4295                 ret = rte_dev_event_callback_register(NULL,
4296                         dev_event_callback, NULL);
4297                 if (ret) {
4298                         RTE_LOG(ERR, EAL,
4299                                 "fail  to register device event callback\n");
4300                         return -1;
4301                 }
4302         }
4303
4304         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4305                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4306
4307         /* set all ports to promiscuous mode by default */
4308         RTE_ETH_FOREACH_DEV(port_id) {
4309                 ret = rte_eth_promiscuous_enable(port_id);
4310                 if (ret != 0)
4311                         fprintf(stderr,
4312                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4313                                 port_id, rte_strerror(-ret));
4314         }
4315
4316 #ifdef RTE_LIB_METRICS
4317         /* Init metrics library */
4318         rte_metrics_init(rte_socket_id());
4319 #endif
4320
4321 #ifdef RTE_LIB_LATENCYSTATS
4322         if (latencystats_enabled != 0) {
4323                 int ret = rte_latencystats_init(1, NULL);
4324                 if (ret)
4325                         fprintf(stderr,
4326                                 "Warning: latencystats init() returned error %d\n",
4327                                 ret);
4328                 fprintf(stderr, "Latencystats running on lcore %d\n",
4329                         latencystats_lcore_id);
4330         }
4331 #endif
4332
4333         /* Setup bitrate stats */
4334 #ifdef RTE_LIB_BITRATESTATS
4335         if (bitrate_enabled != 0) {
4336                 bitrate_data = rte_stats_bitrate_create();
4337                 if (bitrate_data == NULL)
4338                         rte_exit(EXIT_FAILURE,
4339                                 "Could not allocate bitrate data.\n");
4340                 rte_stats_bitrate_reg(bitrate_data);
4341         }
4342 #endif
4343 #ifdef RTE_LIB_CMDLINE
4344         if (strlen(cmdline_filename) != 0)
4345                 cmdline_read_from_file(cmdline_filename);
4346
4347         if (interactive == 1) {
4348                 if (auto_start) {
4349                         printf("Start automatic packet forwarding\n");
4350                         start_packet_forwarding(0);
4351                 }
4352                 prompt();
4353                 pmd_test_exit();
4354         } else
4355 #endif
4356         {
4357                 char c;
4358                 int rc;
4359
4360                 f_quit = 0;
4361
4362                 printf("No commandline core given, start packet forwarding\n");
4363                 start_packet_forwarding(tx_first);
4364                 if (stats_period != 0) {
4365                         uint64_t prev_time = 0, cur_time, diff_time = 0;
4366                         uint64_t timer_period;
4367
4368                         /* Convert to number of cycles */
4369                         timer_period = stats_period * rte_get_timer_hz();
4370
4371                         while (f_quit == 0) {
4372                                 cur_time = rte_get_timer_cycles();
4373                                 diff_time += cur_time - prev_time;
4374
4375                                 if (diff_time >= timer_period) {
4376                                         print_stats();
4377                                         /* Reset the timer */
4378                                         diff_time = 0;
4379                                 }
4380                                 /* Sleep to avoid unnecessary checks */
4381                                 prev_time = cur_time;
4382                                 rte_delay_us_sleep(US_PER_S);
4383                         }
4384                 }
4385
4386                 printf("Press enter to exit\n");
4387                 rc = read(0, &c, 1);
4388                 pmd_test_exit();
4389                 if (rc < 0)
4390                         return 1;
4391         }
4392
4393         ret = rte_eal_cleanup();
4394         if (ret != 0)
4395                 rte_exit(EXIT_FAILURE,
4396                          "EAL cleanup failed: %s\n", strerror(-ret));
4397
4398         return EXIT_SUCCESS;
4399 }