app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_NET_IXGBE
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIB_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #ifdef RTE_LIB_METRICS
  58 #include <rte_metrics.h>
  59 #endif
  60 #ifdef RTE_LIB_BITRATESTATS
  61 #include <rte_bitrate.h>
  62 #endif
  63 #ifdef RTE_LIB_LATENCYSTATS
  64 #include <rte_latencystats.h>
  65 #endif
  66 #ifdef RTE_EXEC_ENV_WINDOWS
  67 #include <process.h>
  68 #endif
  69 #ifdef RTE_NET_BOND
  70 #include <rte_eth_bond.h>
  71 #endif
  72
  73 #include "testpmd.h"
  74
  75 #ifndef MAP_HUGETLB
  76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  77 #define HUGE_FLAG (0x40000)
  78 #else
  79 #define HUGE_FLAG MAP_HUGETLB
  80 #endif
  81
  82 #ifndef MAP_HUGE_SHIFT
  83 /* older kernels (or FreeBSD) will not have this define */
  84 #define HUGE_SHIFT (26)
  85 #else
  86 #define HUGE_SHIFT MAP_HUGE_SHIFT
  87 #endif
  88
  89 #define EXTMEM_HEAP_NAME "extmem"
  90 /*
  91  * Zone size with the malloc overhead (max of debug and release variants)
  92  * must fit into the smallest supported hugepage size (2M),
  93  * so that an IOVA-contiguous zone of this size can always be allocated
  94  * if there are free 2M hugepages.
  95  */
  96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE)
  97
  98 uint16_t verbose_level = 0; /**< Silent by default. */
  99 int testpmd_logtype; /**< Log type for testpmd logs */
 100
 101 /* use main core for command line ? */
 102 uint8_t interactive = 0;
 103 uint8_t auto_start = 0;
 104 uint8_t tx_first;
 105 char cmdline_filename[PATH_MAX] = {0};
 106
 107 /*
 108  * NUMA support configuration.
 109  * When set, the NUMA support attempts to dispatch the allocation of the
 110  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 111  * probed ports among the CPU sockets 0 and 1.
 112  * Otherwise, all memory is allocated from CPU socket 0.
 113  */
 114 uint8_t numa_support = 1; /**< numa enabled by default */
 115
 116 /*
 117  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 118  * not configured.
 119  */
 120 uint8_t socket_num = UMA_NO_CONFIG;
 121
 122 /*
 123  * Select mempool allocation type:
 124  * - native: use regular DPDK memory
 125  * - anon: use regular DPDK memory to create mempool, but populate using
 126  *         anonymous memory (may not be IOVA-contiguous)
 127  * - xmem: use externally allocated hugepage memory
 128  */
 129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 130
 131 /*
 132  * Store specified sockets on which memory pool to be used by ports
 133  * is allocated.
 134  */
 135 uint8_t port_numa[RTE_MAX_ETHPORTS];
 136
 137 /*
 138  * Store specified sockets on which RX ring to be used by ports
 139  * is allocated.
 140  */
 141 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 142
 143 /*
 144  * Store specified sockets on which TX ring to be used by ports
 145  * is allocated.
 146  */
 147 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 148
 149 /*
 150  * Record the Ethernet address of peer target ports to which packets are
 151  * forwarded.
 152  * Must be instantiated with the ethernet addresses of peer traffic generator
 153  * ports.
 154  */
 155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 156 portid_t nb_peer_eth_addrs = 0;
 157
 158 /*
 159  * Probed Target Environment.
 160  */
 161 struct rte_port *ports;        /**< For all probed ethernet ports. */
 162 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 164 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 165
 166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 167
 168 /*
 169  * Test Forwarding Configuration.
 170  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 171  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 172  */
 173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 175 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 176 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 177
 178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 180
 181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 182 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 183
 184 /*
 185  * Forwarding engines.
 186  */
 187 struct fwd_engine * fwd_engines[] = {
 188         &io_fwd_engine,
 189         &mac_fwd_engine,
 190         &mac_swap_engine,
 191         &flow_gen_engine,
 192         &rx_only_engine,
 193         &tx_only_engine,
 194         &csum_fwd_engine,
 195         &icmp_echo_engine,
 196         &noisy_vnf_engine,
 197         &five_tuple_swap_fwd_engine,
 198 #ifdef RTE_LIBRTE_IEEE1588
 199         &ieee1588_fwd_engine,
 200 #endif
 201         &shared_rxq_engine,
 202         NULL,
 203 };
 204
 205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 206 uint16_t mempool_flags;
 207
 208 struct fwd_config cur_fwd_config;
 209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 210 uint32_t retry_enabled;
 211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 213
 214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 216         DEFAULT_MBUF_DATA_SIZE
 217 }; /**< Mbuf data space size. */
 218 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 219                                       * specified on command-line. */
 220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 221
 222 /** Extended statistics to show. */
 223 struct rte_eth_xstat_name *xstats_display;
 224
 225 unsigned int xstats_display_num; /**< Size of extended statistics to show */
 226
 227 /*
 228  * In container, it cannot terminate the process which running with 'stats-period'
 229  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 230  */
 231 uint8_t f_quit;
 232
 233 /*
 234  * Max Rx frame size, set by '--max-pkt-len' parameter.
 235  */
 236 uint32_t max_rx_pkt_len;
 237
 238 /*
 239  * Configuration of packet segments used to scatter received packets
 240  * if some of split features is configured.
 241  */
 242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 243 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 245 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 246
 247 /*
 248  * Configuration of packet segments used by the "txonly" processing engine.
 249  */
 250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 252         TXONLY_DEF_PACKET_LEN,
 253 };
 254 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 255
 256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 257 /**< Split policy for packets to TX. */
 258
 259 uint8_t txonly_multi_flow;
 260 /**< Whether multiple flows are generated in TXONLY mode. */
 261
 262 uint32_t tx_pkt_times_inter;
 263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 264
 265 uint32_t tx_pkt_times_intra;
 266 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 267
 268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
 271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 272
 273 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 274 uint8_t dcb_config = 0;
 275
 276 /*
 277  * Configurable number of RX/TX queues.
 278  */
 279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 282
 283 /*
 284  * Configurable number of RX/TX ring descriptors.
 285  * Defaults are supplied by drivers via ethdev.
 286  */
 287 #define RTE_TEST_RX_DESC_DEFAULT 0
 288 #define RTE_TEST_TX_DESC_DEFAULT 0
 289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 291
 292 #define RTE_PMD_PARAM_UNSET -1
 293 /*
 294  * Configurable values of RX and TX ring threshold registers.
 295  */
 296
 297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 300
 301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 304
 305 /*
 306  * Configurable value of RX free threshold.
 307  */
 308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 309
 310 /*
 311  * Configurable value of RX drop enable.
 312  */
 313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 314
 315 /*
 316  * Configurable value of TX free threshold.
 317  */
 318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 319
 320 /*
 321  * Configurable value of TX RS bit threshold.
 322  */
 323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 324
 325 /*
 326  * Configurable value of buffered packets before sending.
 327  */
 328 uint16_t noisy_tx_sw_bufsz;
 329
 330 /*
 331  * Configurable value of packet buffer timeout.
 332  */
 333 uint16_t noisy_tx_sw_buf_flush_time;
 334
 335 /*
 336  * Configurable value for size of VNF internal memory area
 337  * used for simulating noisy neighbour behaviour
 338  */
 339 uint64_t noisy_lkup_mem_sz;
 340
 341 /*
 342  * Configurable value of number of random writes done in
 343  * VNF simulation memory area.
 344  */
 345 uint64_t noisy_lkup_num_writes;
 346
 347 /*
 348  * Configurable value of number of random reads done in
 349  * VNF simulation memory area.
 350  */
 351 uint64_t noisy_lkup_num_reads;
 352
 353 /*
 354  * Configurable value of number of random reads/writes done in
 355  * VNF simulation memory area.
 356  */
 357 uint64_t noisy_lkup_num_reads_writes;
 358
 359 /*
 360  * Receive Side Scaling (RSS) configuration.
 361  */
 362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */
 363
 364 /*
 365  * Port topology configuration
 366  */
 367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 368
 369 /*
 370  * Avoids to flush all the RX streams before starts forwarding.
 371  */
 372 uint8_t no_flush_rx = 0; /* flush by default */
 373
 374 /*
 375  * Flow API isolated mode.
 376  */
 377 uint8_t flow_isolate_all;
 378
 379 /*
 380  * Avoids to check link status when starting/stopping a port.
 381  */
 382 uint8_t no_link_check = 0; /* check by default */
 383
 384 /*
 385  * Don't automatically start all ports in interactive mode.
 386  */
 387 uint8_t no_device_start = 0;
 388
 389 /*
 390  * Enable link status change notification
 391  */
 392 uint8_t lsc_interrupt = 1; /* enabled by default */
 393
 394 /*
 395  * Enable device removal notification.
 396  */
 397 uint8_t rmv_interrupt = 1; /* enabled by default */
 398
 399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 400
 401 /* After attach, port setup is called on event or by iterator */
 402 bool setup_on_probe_event = true;
 403
 404 /* Clear ptypes on port initialization. */
 405 uint8_t clear_ptypes = true;
 406
 407 /* Hairpin ports configuration mode. */
 408 uint16_t hairpin_mode;
 409
 410 /* Pretty printing of ethdev events */
 411 static const char * const eth_event_desc[] = {
 412         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 413         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 414         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 415         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 416         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 417         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 418         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 419         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 420         [RTE_ETH_EVENT_NEW] = "device probed",
 421         [RTE_ETH_EVENT_DESTROY] = "device released",
 422         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 423         [RTE_ETH_EVENT_MAX] = NULL,
 424 };
 425
 426 /*
 427  * Display or mask ether events
 428  * Default to all events except VF_MBOX
 429  */
 430 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 431                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 432                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 433                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 434                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 435                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 436                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 437                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 438 /*
 439  * Decide if all memory are locked for performance.
 440  */
 441 int do_mlockall = 0;
 442
 443 /*
 444  * NIC bypass mode configuration options.
 445  */
 446
 447 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 448 /* The NIC bypass watchdog timeout. */
 449 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 450 #endif
 451
 452
 453 #ifdef RTE_LIB_LATENCYSTATS
 454
 455 /*
 456  * Set when latency stats is enabled in the commandline
 457  */
 458 uint8_t latencystats_enabled;
 459
 460 /*
 461  * Lcore ID to service latency statistics.
 462  */
 463 lcoreid_t latencystats_lcore_id = -1;
 464
 465 #endif
 466
 467 /*
 468  * Ethernet device configuration.
 469  */
 470 struct rte_eth_rxmode rx_mode;
 471
 472 struct rte_eth_txmode tx_mode = {
 473         .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
 474 };
 475
 476 struct rte_eth_fdir_conf fdir_conf = {
 477         .mode = RTE_FDIR_MODE_NONE,
 478         .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
 479         .status = RTE_FDIR_REPORT_STATUS,
 480         .mask = {
 481                 .vlan_tci_mask = 0xFFEF,
 482                 .ipv4_mask     = {
 483                         .src_ip = 0xFFFFFFFF,
 484                         .dst_ip = 0xFFFFFFFF,
 485                 },
 486                 .ipv6_mask     = {
 487                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 488                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 489                 },
 490                 .src_port_mask = 0xFFFF,
 491                 .dst_port_mask = 0xFFFF,
 492                 .mac_addr_byte_mask = 0xFF,
 493                 .tunnel_type_mask = 1,
 494                 .tunnel_id_mask = 0xFFFFFFFF,
 495         },
 496         .drop_queue = 127,
 497 };
 498
 499 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 500
 501 /*
 502  * Display zero values by default for xstats
 503  */
 504 uint8_t xstats_hide_zero;
 505
 506 /*
 507  * Measure of CPU cycles disabled by default
 508  */
 509 uint8_t record_core_cycles;
 510
 511 /*
 512  * Display of RX and TX bursts disabled by default
 513  */
 514 uint8_t record_burst_stats;
 515
 516 /*
 517  * Number of ports per shared Rx queue group, 0 disable.
 518  */
 519 uint32_t rxq_share;
 520
 521 unsigned int num_sockets = 0;
 522 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 523
 524 #ifdef RTE_LIB_BITRATESTATS
 525 /* Bitrate statistics */
 526 struct rte_stats_bitrates *bitrate_data;
 527 lcoreid_t bitrate_lcore_id;
 528 uint8_t bitrate_enabled;
 529 #endif
 530
 531 #ifdef RTE_LIB_GRO
 532 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 533 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 534 #endif
 535
 536 /*
 537  * hexadecimal bitmask of RX mq mode can be enabled.
 538  */
 539 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
 540
 541 /*
 542  * Used to set forced link speed
 543  */
 544 uint32_t eth_link_speed;
 545
 546 /*
 547  * ID of the current process in multi-process, used to
 548  * configure the queues to be polled.
 549  */
 550 int proc_id;
 551
 552 /*
 553  * Number of processes in multi-process, used to
 554  * configure the queues to be polled.
 555  */
 556 unsigned int num_procs = 1;
 557
 558 static void
 559 eth_rx_metadata_negotiate_mp(uint16_t port_id)
 560 {
 561         uint64_t rx_meta_features = 0;
 562         int ret;
 563
 564         if (!is_proc_primary())
 565                 return;
 566
 567         rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
 568         rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
 569         rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
 570
 571         ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
 572         if (ret == 0) {
 573                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
 574                         TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
 575                                     port_id);
 576                 }
 577
 578                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
 579                         TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
 580                                     port_id);
 581                 }
 582
 583                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
 584                         TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
 585                                     port_id);
 586                 }
 587         } else if (ret != -ENOTSUP) {
 588                 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
 589                          port_id, rte_strerror(-ret));
 590         }
 591 }
 592
 593 static int
 594 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 595                       const struct rte_eth_conf *dev_conf)
 596 {
 597         if (is_proc_primary())
 598                 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
 599                                         dev_conf);
 600         return 0;
 601 }
 602
 603 static int
 604 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop)
 605 {
 606 #ifdef RTE_NET_BOND
 607
 608         portid_t slave_pids[RTE_MAX_ETHPORTS];
 609         struct rte_port *port;
 610         int num_slaves;
 611         portid_t slave_pid;
 612         int i;
 613
 614         num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids,
 615                                                 RTE_MAX_ETHPORTS);
 616         if (num_slaves < 0) {
 617                 fprintf(stderr, "Failed to get slave list for port = %u\n",
 618                         bond_pid);
 619                 return num_slaves;
 620         }
 621
 622         for (i = 0; i < num_slaves; i++) {
 623                 slave_pid = slave_pids[i];
 624                 port = &ports[slave_pid];
 625                 port->port_status =
 626                         is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED;
 627         }
 628 #else
 629         RTE_SET_USED(bond_pid);
 630         RTE_SET_USED(is_stop);
 631 #endif
 632         return 0;
 633 }
 634
 635 static int
 636 eth_dev_start_mp(uint16_t port_id)
 637 {
 638         int ret;
 639
 640         if (is_proc_primary()) {
 641                 ret = rte_eth_dev_start(port_id);
 642                 if (ret != 0)
 643                         return ret;
 644
 645                 struct rte_port *port = &ports[port_id];
 646
 647                 /*
 648                  * Starting a bonded port also starts all slaves under the bonded
 649                  * device. So if this port is bond device, we need to modify the
 650                  * port status of these slaves.
 651                  */
 652                 if (port->bond_flag == 1)
 653                         return change_bonding_slave_port_status(port_id, false);
 654         }
 655
 656         return 0;
 657 }
 658
 659 static int
 660 eth_dev_stop_mp(uint16_t port_id)
 661 {
 662         int ret;
 663
 664         if (is_proc_primary()) {
 665                 ret = rte_eth_dev_stop(port_id);
 666                 if (ret != 0)
 667                         return ret;
 668
 669                 struct rte_port *port = &ports[port_id];
 670
 671                 /*
 672                  * Stopping a bonded port also stops all slaves under the bonded
 673                  * device. So if this port is bond device, we need to modify the
 674                  * port status of these slaves.
 675                  */
 676                 if (port->bond_flag == 1)
 677                         return change_bonding_slave_port_status(port_id, true);
 678         }
 679
 680         return 0;
 681 }
 682
 683 static void
 684 mempool_free_mp(struct rte_mempool *mp)
 685 {
 686         if (is_proc_primary())
 687                 rte_mempool_free(mp);
 688 }
 689
 690 static int
 691 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
 692 {
 693         if (is_proc_primary())
 694                 return rte_eth_dev_set_mtu(port_id, mtu);
 695
 696         return 0;
 697 }
 698
 699 /* Forward function declarations */
 700 static void setup_attached_port(portid_t pi);
 701 static void check_all_ports_link_status(uint32_t port_mask);
 702 static int eth_event_callback(portid_t port_id,
 703                               enum rte_eth_event_type type,
 704                               void *param, void *ret_param);
 705 static void dev_event_callback(const char *device_name,
 706                                 enum rte_dev_event_type type,
 707                                 void *param);
 708 static void fill_xstats_display_info(void);
 709
 710 /*
 711  * Check if all the ports are started.
 712  * If yes, return positive value. If not, return zero.
 713  */
 714 static int all_ports_started(void);
 715
 716 #ifdef RTE_LIB_GSO
 717 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 718 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 719 #endif
 720
 721 /* Holds the registered mbuf dynamic flags names. */
 722 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 723
 724
 725 /*
 726  * Helper function to check if socket is already discovered.
 727  * If yes, return positive value. If not, return zero.
 728  */
 729 int
 730 new_socket_id(unsigned int socket_id)
 731 {
 732         unsigned int i;
 733
 734         for (i = 0; i < num_sockets; i++) {
 735                 if (socket_ids[i] == socket_id)
 736                         return 0;
 737         }
 738         return 1;
 739 }
 740
 741 /*
 742  * Setup default configuration.
 743  */
 744 static void
 745 set_default_fwd_lcores_config(void)
 746 {
 747         unsigned int i;
 748         unsigned int nb_lc;
 749         unsigned int sock_num;
 750
 751         nb_lc = 0;
 752         for (i = 0; i < RTE_MAX_LCORE; i++) {
 753                 if (!rte_lcore_is_enabled(i))
 754                         continue;
 755                 sock_num = rte_lcore_to_socket_id(i);
 756                 if (new_socket_id(sock_num)) {
 757                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 758                                 rte_exit(EXIT_FAILURE,
 759                                          "Total sockets greater than %u\n",
 760                                          RTE_MAX_NUMA_NODES);
 761                         }
 762                         socket_ids[num_sockets++] = sock_num;
 763                 }
 764                 if (i == rte_get_main_lcore())
 765                         continue;
 766                 fwd_lcores_cpuids[nb_lc++] = i;
 767         }
 768         nb_lcores = (lcoreid_t) nb_lc;
 769         nb_cfg_lcores = nb_lcores;
 770         nb_fwd_lcores = 1;
 771 }
 772
 773 static void
 774 set_def_peer_eth_addrs(void)
 775 {
 776         portid_t i;
 777
 778         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 779                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 780                 peer_eth_addrs[i].addr_bytes[5] = i;
 781         }
 782 }
 783
 784 static void
 785 set_default_fwd_ports_config(void)
 786 {
 787         portid_t pt_id;
 788         int i = 0;
 789
 790         RTE_ETH_FOREACH_DEV(pt_id) {
 791                 fwd_ports_ids[i++] = pt_id;
 792
 793                 /* Update sockets info according to the attached device */
 794                 int socket_id = rte_eth_dev_socket_id(pt_id);
 795                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 796                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 797                                 rte_exit(EXIT_FAILURE,
 798                                          "Total sockets greater than %u\n",
 799                                          RTE_MAX_NUMA_NODES);
 800                         }
 801                         socket_ids[num_sockets++] = socket_id;
 802                 }
 803         }
 804
 805         nb_cfg_ports = nb_ports;
 806         nb_fwd_ports = nb_ports;
 807 }
 808
 809 void
 810 set_def_fwd_config(void)
 811 {
 812         set_default_fwd_lcores_config();
 813         set_def_peer_eth_addrs();
 814         set_default_fwd_ports_config();
 815 }
 816
 817 #ifndef RTE_EXEC_ENV_WINDOWS
 818 /* extremely pessimistic estimation of memory required to create a mempool */
 819 static int
 820 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 821 {
 822         unsigned int n_pages, mbuf_per_pg, leftover;
 823         uint64_t total_mem, mbuf_mem, obj_sz;
 824
 825         /* there is no good way to predict how much space the mempool will
 826          * occupy because it will allocate chunks on the fly, and some of those
 827          * will come from default DPDK memory while some will come from our
 828          * external memory, so just assume 128MB will be enough for everyone.
 829          */
 830         uint64_t hdr_mem = 128 << 20;
 831
 832         /* account for possible non-contiguousness */
 833         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 834         if (obj_sz > pgsz) {
 835                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 836                 return -1;
 837         }
 838
 839         mbuf_per_pg = pgsz / obj_sz;
 840         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 841         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 842
 843         mbuf_mem = n_pages * pgsz;
 844
 845         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 846
 847         if (total_mem > SIZE_MAX) {
 848                 TESTPMD_LOG(ERR, "Memory size too big\n");
 849                 return -1;
 850         }
 851         *out = (size_t)total_mem;
 852
 853         return 0;
 854 }
 855
 856 static int
 857 pagesz_flags(uint64_t page_sz)
 858 {
 859         /* as per mmap() manpage, all page sizes are log2 of page size
 860          * shifted by MAP_HUGE_SHIFT
 861          */
 862         int log2 = rte_log2_u64(page_sz);
 863
 864         return (log2 << HUGE_SHIFT);
 865 }
 866
 867 static void *
 868 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 869 {
 870         void *addr;
 871         int flags;
 872
 873         /* allocate anonymous hugepages */
 874         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 875         if (huge)
 876                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 877
 878         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 879         if (addr == MAP_FAILED)
 880                 return NULL;
 881
 882         return addr;
 883 }
 884
 885 struct extmem_param {
 886         void *addr;
 887         size_t len;
 888         size_t pgsz;
 889         rte_iova_t *iova_table;
 890         unsigned int iova_table_len;
 891 };
 892
 893 static int
 894 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 895                 bool huge)
 896 {
 897         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 898                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 899         unsigned int cur_page, n_pages, pgsz_idx;
 900         size_t mem_sz, cur_pgsz;
 901         rte_iova_t *iovas = NULL;
 902         void *addr;
 903         int ret;
 904
 905         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 906                 /* skip anything that is too big */
 907                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 908                         continue;
 909
 910                 cur_pgsz = pgsizes[pgsz_idx];
 911
 912                 /* if we were told not to allocate hugepages, override */
 913                 if (!huge)
 914                         cur_pgsz = sysconf(_SC_PAGESIZE);
 915
 916                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 917                 if (ret < 0) {
 918                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 919                         return -1;
 920                 }
 921
 922                 /* allocate our memory */
 923                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 924
 925                 /* if we couldn't allocate memory with a specified page size,
 926                  * that doesn't mean we can't do it with other page sizes, so
 927                  * try another one.
 928                  */
 929                 if (addr == NULL)
 930                         continue;
 931
 932                 /* store IOVA addresses for every page in this memory area */
 933                 n_pages = mem_sz / cur_pgsz;
 934
 935                 iovas = malloc(sizeof(*iovas) * n_pages);
 936
 937                 if (iovas == NULL) {
 938                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 939                         goto fail;
 940                 }
 941                 /* lock memory if it's not huge pages */
 942                 if (!huge)
 943                         mlock(addr, mem_sz);
 944
 945                 /* populate IOVA addresses */
 946                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 947                         rte_iova_t iova;
 948                         size_t offset;
 949                         void *cur;
 950
 951                         offset = cur_pgsz * cur_page;
 952                         cur = RTE_PTR_ADD(addr, offset);
 953
 954                         /* touch the page before getting its IOVA */
 955                         *(volatile char *)cur = 0;
 956
 957                         iova = rte_mem_virt2iova(cur);
 958
 959                         iovas[cur_page] = iova;
 960                 }
 961
 962                 break;
 963         }
 964         /* if we couldn't allocate anything */
 965         if (iovas == NULL)
 966                 return -1;
 967
 968         param->addr = addr;
 969         param->len = mem_sz;
 970         param->pgsz = cur_pgsz;
 971         param->iova_table = iovas;
 972         param->iova_table_len = n_pages;
 973
 974         return 0;
 975 fail:
 976         free(iovas);
 977         if (addr)
 978                 munmap(addr, mem_sz);
 979
 980         return -1;
 981 }
 982
 983 static int
 984 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 985 {
 986         struct extmem_param param;
 987         int socket_id, ret;
 988
 989         memset(&param, 0, sizeof(param));
 990
 991         /* check if our heap exists */
 992         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 993         if (socket_id < 0) {
 994                 /* create our heap */
 995                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 996                 if (ret < 0) {
 997                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 998                         return -1;
 999                 }
1000         }
1001
1002         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
1003         if (ret < 0) {
1004                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
1005                 return -1;
1006         }
1007
1008         /* we now have a valid memory area, so add it to heap */
1009         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
1010                         param.addr, param.len, param.iova_table,
1011                         param.iova_table_len, param.pgsz);
1012
1013         /* when using VFIO, memory is automatically mapped for DMA by EAL */
1014
1015         /* not needed any more */
1016         free(param.iova_table);
1017
1018         if (ret < 0) {
1019                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
1020                 munmap(param.addr, param.len);
1021                 return -1;
1022         }
1023
1024         /* success */
1025
1026         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
1027                         param.len >> 20);
1028
1029         return 0;
1030 }
1031 static void
1032 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1033              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1034 {
1035         uint16_t pid = 0;
1036         int ret;
1037
1038         RTE_ETH_FOREACH_DEV(pid) {
1039                 struct rte_eth_dev_info dev_info;
1040
1041                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1042                 if (ret != 0) {
1043                         TESTPMD_LOG(DEBUG,
1044                                     "unable to get device info for port %d on addr 0x%p,"
1045                                     "mempool unmapping will not be performed\n",
1046                                     pid, memhdr->addr);
1047                         continue;
1048                 }
1049
1050                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
1051                 if (ret) {
1052                         TESTPMD_LOG(DEBUG,
1053                                     "unable to DMA unmap addr 0x%p "
1054                                     "for device %s\n",
1055                                     memhdr->addr, dev_info.device->name);
1056                 }
1057         }
1058         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
1059         if (ret) {
1060                 TESTPMD_LOG(DEBUG,
1061                             "unable to un-register addr 0x%p\n", memhdr->addr);
1062         }
1063 }
1064
1065 static void
1066 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1067            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1068 {
1069         uint16_t pid = 0;
1070         size_t page_size = sysconf(_SC_PAGESIZE);
1071         int ret;
1072
1073         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1074                                   page_size);
1075         if (ret) {
1076                 TESTPMD_LOG(DEBUG,
1077                             "unable to register addr 0x%p\n", memhdr->addr);
1078                 return;
1079         }
1080         RTE_ETH_FOREACH_DEV(pid) {
1081                 struct rte_eth_dev_info dev_info;
1082
1083                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1084                 if (ret != 0) {
1085                         TESTPMD_LOG(DEBUG,
1086                                     "unable to get device info for port %d on addr 0x%p,"
1087                                     "mempool mapping will not be performed\n",
1088                                     pid, memhdr->addr);
1089                         continue;
1090                 }
1091                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1092                 if (ret) {
1093                         TESTPMD_LOG(DEBUG,
1094                                     "unable to DMA map addr 0x%p "
1095                                     "for device %s\n",
1096                                     memhdr->addr, dev_info.device->name);
1097                 }
1098         }
1099 }
1100 #endif
1101
1102 static unsigned int
1103 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1104             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1105 {
1106         struct rte_pktmbuf_extmem *xmem;
1107         unsigned int ext_num, zone_num, elt_num;
1108         uint16_t elt_size;
1109
1110         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1111         elt_num = EXTBUF_ZONE_SIZE / elt_size;
1112         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1113
1114         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1115         if (xmem == NULL) {
1116                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1117                                  "external buffer descriptors\n");
1118                 *ext_mem = NULL;
1119                 return 0;
1120         }
1121         for (ext_num = 0; ext_num < zone_num; ext_num++) {
1122                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1123                 const struct rte_memzone *mz;
1124                 char mz_name[RTE_MEMZONE_NAMESIZE];
1125                 int ret;
1126
1127                 ret = snprintf(mz_name, sizeof(mz_name),
1128                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1129                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1130                         errno = ENAMETOOLONG;
1131                         ext_num = 0;
1132                         break;
1133                 }
1134                 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE,
1135                                          socket_id,
1136                                          RTE_MEMZONE_IOVA_CONTIG |
1137                                          RTE_MEMZONE_1GB |
1138                                          RTE_MEMZONE_SIZE_HINT_ONLY);
1139                 if (mz == NULL) {
1140                         /*
1141                          * The caller exits on external buffer creation
1142                          * error, so there is no need to free memzones.
1143                          */
1144                         errno = ENOMEM;
1145                         ext_num = 0;
1146                         break;
1147                 }
1148                 xseg->buf_ptr = mz->addr;
1149                 xseg->buf_iova = mz->iova;
1150                 xseg->buf_len = EXTBUF_ZONE_SIZE;
1151                 xseg->elt_size = elt_size;
1152         }
1153         if (ext_num == 0 && xmem != NULL) {
1154                 free(xmem);
1155                 xmem = NULL;
1156         }
1157         *ext_mem = xmem;
1158         return ext_num;
1159 }
1160
1161 /*
1162  * Configuration initialisation done once at init time.
1163  */
1164 static struct rte_mempool *
1165 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1166                  unsigned int socket_id, uint16_t size_idx)
1167 {
1168         char pool_name[RTE_MEMPOOL_NAMESIZE];
1169         struct rte_mempool *rte_mp = NULL;
1170 #ifndef RTE_EXEC_ENV_WINDOWS
1171         uint32_t mb_size;
1172
1173         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1174 #endif
1175         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1176         if (!is_proc_primary()) {
1177                 rte_mp = rte_mempool_lookup(pool_name);
1178                 if (rte_mp == NULL)
1179                         rte_exit(EXIT_FAILURE,
1180                                 "Get mbuf pool for socket %u failed: %s\n",
1181                                 socket_id, rte_strerror(rte_errno));
1182                 return rte_mp;
1183         }
1184
1185         TESTPMD_LOG(INFO,
1186                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1187                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1188
1189         switch (mp_alloc_type) {
1190         case MP_ALLOC_NATIVE:
1191                 {
1192                         /* wrapper to rte_mempool_create() */
1193                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1194                                         rte_mbuf_best_mempool_ops());
1195                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1196                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1197                         break;
1198                 }
1199 #ifndef RTE_EXEC_ENV_WINDOWS
1200         case MP_ALLOC_ANON:
1201                 {
1202                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1203                                 mb_size, (unsigned int) mb_mempool_cache,
1204                                 sizeof(struct rte_pktmbuf_pool_private),
1205                                 socket_id, mempool_flags);
1206                         if (rte_mp == NULL)
1207                                 goto err;
1208
1209                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1210                                 rte_mempool_free(rte_mp);
1211                                 rte_mp = NULL;
1212                                 goto err;
1213                         }
1214                         rte_pktmbuf_pool_init(rte_mp, NULL);
1215                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1216                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1217                         break;
1218                 }
1219         case MP_ALLOC_XMEM:
1220         case MP_ALLOC_XMEM_HUGE:
1221                 {
1222                         int heap_socket;
1223                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1224
1225                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1226                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1227
1228                         heap_socket =
1229                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1230                         if (heap_socket < 0)
1231                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1232
1233                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1234                                         rte_mbuf_best_mempool_ops());
1235                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1236                                         mb_mempool_cache, 0, mbuf_seg_size,
1237                                         heap_socket);
1238                         break;
1239                 }
1240 #endif
1241         case MP_ALLOC_XBUF:
1242                 {
1243                         struct rte_pktmbuf_extmem *ext_mem;
1244                         unsigned int ext_num;
1245
1246                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1247                                                socket_id, pool_name, &ext_mem);
1248                         if (ext_num == 0)
1249                                 rte_exit(EXIT_FAILURE,
1250                                          "Can't create pinned data buffers\n");
1251
1252                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1253                                         rte_mbuf_best_mempool_ops());
1254                         rte_mp = rte_pktmbuf_pool_create_extbuf
1255                                         (pool_name, nb_mbuf, mb_mempool_cache,
1256                                          0, mbuf_seg_size, socket_id,
1257                                          ext_mem, ext_num);
1258                         free(ext_mem);
1259                         break;
1260                 }
1261         default:
1262                 {
1263                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1264                 }
1265         }
1266
1267 #ifndef RTE_EXEC_ENV_WINDOWS
1268 err:
1269 #endif
1270         if (rte_mp == NULL) {
1271                 rte_exit(EXIT_FAILURE,
1272                         "Creation of mbuf pool for socket %u failed: %s\n",
1273                         socket_id, rte_strerror(rte_errno));
1274         } else if (verbose_level > 0) {
1275                 rte_mempool_dump(stdout, rte_mp);
1276         }
1277         return rte_mp;
1278 }
1279
1280 /*
1281  * Check given socket id is valid or not with NUMA mode,
1282  * if valid, return 0, else return -1
1283  */
1284 static int
1285 check_socket_id(const unsigned int socket_id)
1286 {
1287         static int warning_once = 0;
1288
1289         if (new_socket_id(socket_id)) {
1290                 if (!warning_once && numa_support)
1291                         fprintf(stderr,
1292                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1293                 warning_once = 1;
1294                 return -1;
1295         }
1296         return 0;
1297 }
1298
1299 /*
1300  * Get the allowed maximum number of RX queues.
1301  * *pid return the port id which has minimal value of
1302  * max_rx_queues in all ports.
1303  */
1304 queueid_t
1305 get_allowed_max_nb_rxq(portid_t *pid)
1306 {
1307         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1308         bool max_rxq_valid = false;
1309         portid_t pi;
1310         struct rte_eth_dev_info dev_info;
1311
1312         RTE_ETH_FOREACH_DEV(pi) {
1313                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1314                         continue;
1315
1316                 max_rxq_valid = true;
1317                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1318                         allowed_max_rxq = dev_info.max_rx_queues;
1319                         *pid = pi;
1320                 }
1321         }
1322         return max_rxq_valid ? allowed_max_rxq : 0;
1323 }
1324
1325 /*
1326  * Check input rxq is valid or not.
1327  * If input rxq is not greater than any of maximum number
1328  * of RX queues of all ports, it is valid.
1329  * if valid, return 0, else return -1
1330  */
1331 int
1332 check_nb_rxq(queueid_t rxq)
1333 {
1334         queueid_t allowed_max_rxq;
1335         portid_t pid = 0;
1336
1337         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1338         if (rxq > allowed_max_rxq) {
1339                 fprintf(stderr,
1340                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1341                         rxq, allowed_max_rxq, pid);
1342                 return -1;
1343         }
1344         return 0;
1345 }
1346
1347 /*
1348  * Get the allowed maximum number of TX queues.
1349  * *pid return the port id which has minimal value of
1350  * max_tx_queues in all ports.
1351  */
1352 queueid_t
1353 get_allowed_max_nb_txq(portid_t *pid)
1354 {
1355         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1356         bool max_txq_valid = false;
1357         portid_t pi;
1358         struct rte_eth_dev_info dev_info;
1359
1360         RTE_ETH_FOREACH_DEV(pi) {
1361                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1362                         continue;
1363
1364                 max_txq_valid = true;
1365                 if (dev_info.max_tx_queues < allowed_max_txq) {
1366                         allowed_max_txq = dev_info.max_tx_queues;
1367                         *pid = pi;
1368                 }
1369         }
1370         return max_txq_valid ? allowed_max_txq : 0;
1371 }
1372
1373 /*
1374  * Check input txq is valid or not.
1375  * If input txq is not greater than any of maximum number
1376  * of TX queues of all ports, it is valid.
1377  * if valid, return 0, else return -1
1378  */
1379 int
1380 check_nb_txq(queueid_t txq)
1381 {
1382         queueid_t allowed_max_txq;
1383         portid_t pid = 0;
1384
1385         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1386         if (txq > allowed_max_txq) {
1387                 fprintf(stderr,
1388                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1389                         txq, allowed_max_txq, pid);
1390                 return -1;
1391         }
1392         return 0;
1393 }
1394
1395 /*
1396  * Get the allowed maximum number of RXDs of every rx queue.
1397  * *pid return the port id which has minimal value of
1398  * max_rxd in all queues of all ports.
1399  */
1400 static uint16_t
1401 get_allowed_max_nb_rxd(portid_t *pid)
1402 {
1403         uint16_t allowed_max_rxd = UINT16_MAX;
1404         portid_t pi;
1405         struct rte_eth_dev_info dev_info;
1406
1407         RTE_ETH_FOREACH_DEV(pi) {
1408                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1409                         continue;
1410
1411                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1412                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1413                         *pid = pi;
1414                 }
1415         }
1416         return allowed_max_rxd;
1417 }
1418
1419 /*
1420  * Get the allowed minimal number of RXDs of every rx queue.
1421  * *pid return the port id which has minimal value of
1422  * min_rxd in all queues of all ports.
1423  */
1424 static uint16_t
1425 get_allowed_min_nb_rxd(portid_t *pid)
1426 {
1427         uint16_t allowed_min_rxd = 0;
1428         portid_t pi;
1429         struct rte_eth_dev_info dev_info;
1430
1431         RTE_ETH_FOREACH_DEV(pi) {
1432                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1433                         continue;
1434
1435                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1436                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1437                         *pid = pi;
1438                 }
1439         }
1440
1441         return allowed_min_rxd;
1442 }
1443
1444 /*
1445  * Check input rxd is valid or not.
1446  * If input rxd is not greater than any of maximum number
1447  * of RXDs of every Rx queues and is not less than any of
1448  * minimal number of RXDs of every Rx queues, it is valid.
1449  * if valid, return 0, else return -1
1450  */
1451 int
1452 check_nb_rxd(queueid_t rxd)
1453 {
1454         uint16_t allowed_max_rxd;
1455         uint16_t allowed_min_rxd;
1456         portid_t pid = 0;
1457
1458         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1459         if (rxd > allowed_max_rxd) {
1460                 fprintf(stderr,
1461                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1462                         rxd, allowed_max_rxd, pid);
1463                 return -1;
1464         }
1465
1466         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1467         if (rxd < allowed_min_rxd) {
1468                 fprintf(stderr,
1469                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1470                         rxd, allowed_min_rxd, pid);
1471                 return -1;
1472         }
1473
1474         return 0;
1475 }
1476
1477 /*
1478  * Get the allowed maximum number of TXDs of every rx queues.
1479  * *pid return the port id which has minimal value of
1480  * max_txd in every tx queue.
1481  */
1482 static uint16_t
1483 get_allowed_max_nb_txd(portid_t *pid)
1484 {
1485         uint16_t allowed_max_txd = UINT16_MAX;
1486         portid_t pi;
1487         struct rte_eth_dev_info dev_info;
1488
1489         RTE_ETH_FOREACH_DEV(pi) {
1490                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1491                         continue;
1492
1493                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1494                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1495                         *pid = pi;
1496                 }
1497         }
1498         return allowed_max_txd;
1499 }
1500
1501 /*
1502  * Get the allowed maximum number of TXDs of every tx queues.
1503  * *pid return the port id which has minimal value of
1504  * min_txd in every tx queue.
1505  */
1506 static uint16_t
1507 get_allowed_min_nb_txd(portid_t *pid)
1508 {
1509         uint16_t allowed_min_txd = 0;
1510         portid_t pi;
1511         struct rte_eth_dev_info dev_info;
1512
1513         RTE_ETH_FOREACH_DEV(pi) {
1514                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1515                         continue;
1516
1517                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1518                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1519                         *pid = pi;
1520                 }
1521         }
1522
1523         return allowed_min_txd;
1524 }
1525
1526 /*
1527  * Check input txd is valid or not.
1528  * If input txd is not greater than any of maximum number
1529  * of TXDs of every Rx queues, it is valid.
1530  * if valid, return 0, else return -1
1531  */
1532 int
1533 check_nb_txd(queueid_t txd)
1534 {
1535         uint16_t allowed_max_txd;
1536         uint16_t allowed_min_txd;
1537         portid_t pid = 0;
1538
1539         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1540         if (txd > allowed_max_txd) {
1541                 fprintf(stderr,
1542                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1543                         txd, allowed_max_txd, pid);
1544                 return -1;
1545         }
1546
1547         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1548         if (txd < allowed_min_txd) {
1549                 fprintf(stderr,
1550                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1551                         txd, allowed_min_txd, pid);
1552                 return -1;
1553         }
1554         return 0;
1555 }
1556
1557
1558 /*
1559  * Get the allowed maximum number of hairpin queues.
1560  * *pid return the port id which has minimal value of
1561  * max_hairpin_queues in all ports.
1562  */
1563 queueid_t
1564 get_allowed_max_nb_hairpinq(portid_t *pid)
1565 {
1566         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1567         portid_t pi;
1568         struct rte_eth_hairpin_cap cap;
1569
1570         RTE_ETH_FOREACH_DEV(pi) {
1571                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1572                         *pid = pi;
1573                         return 0;
1574                 }
1575                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1576                         allowed_max_hairpinq = cap.max_nb_queues;
1577                         *pid = pi;
1578                 }
1579         }
1580         return allowed_max_hairpinq;
1581 }
1582
1583 /*
1584  * Check input hairpin is valid or not.
1585  * If input hairpin is not greater than any of maximum number
1586  * of hairpin queues of all ports, it is valid.
1587  * if valid, return 0, else return -1
1588  */
1589 int
1590 check_nb_hairpinq(queueid_t hairpinq)
1591 {
1592         queueid_t allowed_max_hairpinq;
1593         portid_t pid = 0;
1594
1595         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1596         if (hairpinq > allowed_max_hairpinq) {
1597                 fprintf(stderr,
1598                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1599                         hairpinq, allowed_max_hairpinq, pid);
1600                 return -1;
1601         }
1602         return 0;
1603 }
1604
1605 static int
1606 get_eth_overhead(struct rte_eth_dev_info *dev_info)
1607 {
1608         uint32_t eth_overhead;
1609
1610         if (dev_info->max_mtu != UINT16_MAX &&
1611             dev_info->max_rx_pktlen > dev_info->max_mtu)
1612                 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1613         else
1614                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1615
1616         return eth_overhead;
1617 }
1618
1619 static void
1620 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1621 {
1622         struct rte_port *port = &ports[pid];
1623         int ret;
1624         int i;
1625
1626         eth_rx_metadata_negotiate_mp(pid);
1627
1628         port->dev_conf.txmode = tx_mode;
1629         port->dev_conf.rxmode = rx_mode;
1630
1631         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1632         if (ret != 0)
1633                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1634
1635         if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1636                 port->dev_conf.txmode.offloads &=
1637                         ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1638
1639         /* Apply Rx offloads configuration */
1640         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1641                 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads;
1642         /* Apply Tx offloads configuration */
1643         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1644                 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads;
1645
1646         if (eth_link_speed)
1647                 port->dev_conf.link_speeds = eth_link_speed;
1648
1649         if (max_rx_pkt_len)
1650                 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1651                         get_eth_overhead(&port->dev_info);
1652
1653         /* set flag to initialize port/queue */
1654         port->need_reconfig = 1;
1655         port->need_reconfig_queues = 1;
1656         port->socket_id = socket_id;
1657         port->tx_metadata = 0;
1658
1659         /*
1660          * Check for maximum number of segments per MTU.
1661          * Accordingly update the mbuf data size.
1662          */
1663         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1664             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1665                 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1666                 uint16_t mtu;
1667
1668                 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1669                         uint16_t data_size = (mtu + eth_overhead) /
1670                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1671                         uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1672
1673                         if (buffer_size > mbuf_data_size[0]) {
1674                                 mbuf_data_size[0] = buffer_size;
1675                                 TESTPMD_LOG(WARNING,
1676                                         "Configured mbuf size of the first segment %hu\n",
1677                                         mbuf_data_size[0]);
1678                         }
1679                 }
1680         }
1681 }
1682
1683 static void
1684 init_config(void)
1685 {
1686         portid_t pid;
1687         struct rte_mempool *mbp;
1688         unsigned int nb_mbuf_per_pool;
1689         lcoreid_t  lc_id;
1690 #ifdef RTE_LIB_GRO
1691         struct rte_gro_param gro_param;
1692 #endif
1693 #ifdef RTE_LIB_GSO
1694         uint32_t gso_types;
1695 #endif
1696
1697         /* Configuration of logical cores. */
1698         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1699                                 sizeof(struct fwd_lcore *) * nb_lcores,
1700                                 RTE_CACHE_LINE_SIZE);
1701         if (fwd_lcores == NULL) {
1702                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1703                                                         "failed\n", nb_lcores);
1704         }
1705         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1706                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1707                                                sizeof(struct fwd_lcore),
1708                                                RTE_CACHE_LINE_SIZE);
1709                 if (fwd_lcores[lc_id] == NULL) {
1710                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1711                                                                 "failed\n");
1712                 }
1713                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1714         }
1715
1716         RTE_ETH_FOREACH_DEV(pid) {
1717                 uint32_t socket_id;
1718
1719                 if (numa_support) {
1720                         socket_id = port_numa[pid];
1721                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1722                                 socket_id = rte_eth_dev_socket_id(pid);
1723
1724                                 /*
1725                                  * if socket_id is invalid,
1726                                  * set to the first available socket.
1727                                  */
1728                                 if (check_socket_id(socket_id) < 0)
1729                                         socket_id = socket_ids[0];
1730                         }
1731                 } else {
1732                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1733                                     0 : socket_num;
1734                 }
1735                 /* Apply default TxRx configuration for all ports */
1736                 init_config_port_offloads(pid, socket_id);
1737         }
1738         /*
1739          * Create pools of mbuf.
1740          * If NUMA support is disabled, create a single pool of mbuf in
1741          * socket 0 memory by default.
1742          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1743          *
1744          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1745          * nb_txd can be configured at run time.
1746          */
1747         if (param_total_num_mbufs)
1748                 nb_mbuf_per_pool = param_total_num_mbufs;
1749         else {
1750                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1751                         (nb_lcores * mb_mempool_cache) +
1752                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1753                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1754         }
1755
1756         if (numa_support) {
1757                 uint8_t i, j;
1758
1759                 for (i = 0; i < num_sockets; i++)
1760                         for (j = 0; j < mbuf_data_size_n; j++)
1761                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1762                                         mbuf_pool_create(mbuf_data_size[j],
1763                                                           nb_mbuf_per_pool,
1764                                                           socket_ids[i], j);
1765         } else {
1766                 uint8_t i;
1767
1768                 for (i = 0; i < mbuf_data_size_n; i++)
1769                         mempools[i] = mbuf_pool_create
1770                                         (mbuf_data_size[i],
1771                                          nb_mbuf_per_pool,
1772                                          socket_num == UMA_NO_CONFIG ?
1773                                          0 : socket_num, i);
1774         }
1775
1776         init_port_config();
1777
1778 #ifdef RTE_LIB_GSO
1779         gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1780                 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1781 #endif
1782         /*
1783          * Records which Mbuf pool to use by each logical core, if needed.
1784          */
1785         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1786                 mbp = mbuf_pool_find(
1787                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1788
1789                 if (mbp == NULL)
1790                         mbp = mbuf_pool_find(0, 0);
1791                 fwd_lcores[lc_id]->mbp = mbp;
1792 #ifdef RTE_LIB_GSO
1793                 /* initialize GSO context */
1794                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1795                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1796                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1797                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1798                         RTE_ETHER_CRC_LEN;
1799                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1800 #endif
1801         }
1802
1803         fwd_config_setup();
1804
1805 #ifdef RTE_LIB_GRO
1806         /* create a gro context for each lcore */
1807         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1808         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1809         gro_param.max_item_per_flow = MAX_PKT_BURST;
1810         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1811                 gro_param.socket_id = rte_lcore_to_socket_id(
1812                                 fwd_lcores_cpuids[lc_id]);
1813                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1814                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1815                         rte_exit(EXIT_FAILURE,
1816                                         "rte_gro_ctx_create() failed\n");
1817                 }
1818         }
1819 #endif
1820 }
1821
1822
1823 void
1824 reconfig(portid_t new_port_id, unsigned socket_id)
1825 {
1826         /* Reconfiguration of Ethernet ports. */
1827         init_config_port_offloads(new_port_id, socket_id);
1828         init_port_config();
1829 }
1830
1831 int
1832 init_fwd_streams(void)
1833 {
1834         portid_t pid;
1835         struct rte_port *port;
1836         streamid_t sm_id, nb_fwd_streams_new;
1837         queueid_t q;
1838
1839         /* set socket id according to numa or not */
1840         RTE_ETH_FOREACH_DEV(pid) {
1841                 port = &ports[pid];
1842                 if (nb_rxq > port->dev_info.max_rx_queues) {
1843                         fprintf(stderr,
1844                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1845                                 nb_rxq, port->dev_info.max_rx_queues);
1846                         return -1;
1847                 }
1848                 if (nb_txq > port->dev_info.max_tx_queues) {
1849                         fprintf(stderr,
1850                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1851                                 nb_txq, port->dev_info.max_tx_queues);
1852                         return -1;
1853                 }
1854                 if (numa_support) {
1855                         if (port_numa[pid] != NUMA_NO_CONFIG)
1856                                 port->socket_id = port_numa[pid];
1857                         else {
1858                                 port->socket_id = rte_eth_dev_socket_id(pid);
1859
1860                                 /*
1861                                  * if socket_id is invalid,
1862                                  * set to the first available socket.
1863                                  */
1864                                 if (check_socket_id(port->socket_id) < 0)
1865                                         port->socket_id = socket_ids[0];
1866                         }
1867                 }
1868                 else {
1869                         if (socket_num == UMA_NO_CONFIG)
1870                                 port->socket_id = 0;
1871                         else
1872                                 port->socket_id = socket_num;
1873                 }
1874         }
1875
1876         q = RTE_MAX(nb_rxq, nb_txq);
1877         if (q == 0) {
1878                 fprintf(stderr,
1879                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1880                 return -1;
1881         }
1882         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1883         if (nb_fwd_streams_new == nb_fwd_streams)
1884                 return 0;
1885         /* clear the old */
1886         if (fwd_streams != NULL) {
1887                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1888                         if (fwd_streams[sm_id] == NULL)
1889                                 continue;
1890                         rte_free(fwd_streams[sm_id]);
1891                         fwd_streams[sm_id] = NULL;
1892                 }
1893                 rte_free(fwd_streams);
1894                 fwd_streams = NULL;
1895         }
1896
1897         /* init new */
1898         nb_fwd_streams = nb_fwd_streams_new;
1899         if (nb_fwd_streams) {
1900                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1901                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1902                         RTE_CACHE_LINE_SIZE);
1903                 if (fwd_streams == NULL)
1904                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1905                                  " (struct fwd_stream *)) failed\n",
1906                                  nb_fwd_streams);
1907
1908                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1909                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1910                                 " struct fwd_stream", sizeof(struct fwd_stream),
1911                                 RTE_CACHE_LINE_SIZE);
1912                         if (fwd_streams[sm_id] == NULL)
1913                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1914                                          "(struct fwd_stream) failed\n");
1915                 }
1916         }
1917
1918         return 0;
1919 }
1920
1921 static void
1922 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1923 {
1924         uint64_t total_burst, sburst;
1925         uint64_t nb_burst;
1926         uint64_t burst_stats[4];
1927         uint16_t pktnb_stats[4];
1928         uint16_t nb_pkt;
1929         int burst_percent[4], sburstp;
1930         int i;
1931
1932         /*
1933          * First compute the total number of packet bursts and the
1934          * two highest numbers of bursts of the same number of packets.
1935          */
1936         memset(&burst_stats, 0x0, sizeof(burst_stats));
1937         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1938
1939         /* Show stats for 0 burst size always */
1940         total_burst = pbs->pkt_burst_spread[0];
1941         burst_stats[0] = pbs->pkt_burst_spread[0];
1942         pktnb_stats[0] = 0;
1943
1944         /* Find the next 2 burst sizes with highest occurrences. */
1945         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1946                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1947
1948                 if (nb_burst == 0)
1949                         continue;
1950
1951                 total_burst += nb_burst;
1952
1953                 if (nb_burst > burst_stats[1]) {
1954                         burst_stats[2] = burst_stats[1];
1955                         pktnb_stats[2] = pktnb_stats[1];
1956                         burst_stats[1] = nb_burst;
1957                         pktnb_stats[1] = nb_pkt;
1958                 } else if (nb_burst > burst_stats[2]) {
1959                         burst_stats[2] = nb_burst;
1960                         pktnb_stats[2] = nb_pkt;
1961                 }
1962         }
1963         if (total_burst == 0)
1964                 return;
1965
1966         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1967         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1968                 if (i == 3) {
1969                         printf("%d%% of other]\n", 100 - sburstp);
1970                         return;
1971                 }
1972
1973                 sburst += burst_stats[i];
1974                 if (sburst == total_burst) {
1975                         printf("%d%% of %d pkts]\n",
1976                                 100 - sburstp, (int) pktnb_stats[i]);
1977                         return;
1978                 }
1979
1980                 burst_percent[i] =
1981                         (double)burst_stats[i] / total_burst * 100;
1982                 printf("%d%% of %d pkts + ",
1983                         burst_percent[i], (int) pktnb_stats[i]);
1984                 sburstp += burst_percent[i];
1985         }
1986 }
1987
1988 static void
1989 fwd_stream_stats_display(streamid_t stream_id)
1990 {
1991         struct fwd_stream *fs;
1992         static const char *fwd_top_stats_border = "-------";
1993
1994         fs = fwd_streams[stream_id];
1995         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1996             (fs->fwd_dropped == 0))
1997                 return;
1998         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1999                "TX Port=%2d/Queue=%2d %s\n",
2000                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
2001                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
2002         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
2003                " TX-dropped: %-14"PRIu64,
2004                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
2005
2006         /* if checksum mode */
2007         if (cur_fwd_eng == &csum_fwd_engine) {
2008                 printf("  RX- bad IP checksum: %-14"PRIu64
2009                        "  Rx- bad L4 checksum: %-14"PRIu64
2010                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
2011                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
2012                         fs->rx_bad_outer_l4_csum);
2013                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
2014                         fs->rx_bad_outer_ip_csum);
2015         } else {
2016                 printf("\n");
2017         }
2018
2019         if (record_burst_stats) {
2020                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
2021                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
2022         }
2023 }
2024
2025 void
2026 fwd_stats_display(void)
2027 {
2028         static const char *fwd_stats_border = "----------------------";
2029         static const char *acc_stats_border = "+++++++++++++++";
2030         struct {
2031                 struct fwd_stream *rx_stream;
2032                 struct fwd_stream *tx_stream;
2033                 uint64_t tx_dropped;
2034                 uint64_t rx_bad_ip_csum;
2035                 uint64_t rx_bad_l4_csum;
2036                 uint64_t rx_bad_outer_l4_csum;
2037                 uint64_t rx_bad_outer_ip_csum;
2038         } ports_stats[RTE_MAX_ETHPORTS];
2039         uint64_t total_rx_dropped = 0;
2040         uint64_t total_tx_dropped = 0;
2041         uint64_t total_rx_nombuf = 0;
2042         struct rte_eth_stats stats;
2043         uint64_t fwd_cycles = 0;
2044         uint64_t total_recv = 0;
2045         uint64_t total_xmit = 0;
2046         struct rte_port *port;
2047         streamid_t sm_id;
2048         portid_t pt_id;
2049         int ret;
2050         int i;
2051
2052         memset(ports_stats, 0, sizeof(ports_stats));
2053
2054         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2055                 struct fwd_stream *fs = fwd_streams[sm_id];
2056
2057                 if (cur_fwd_config.nb_fwd_streams >
2058                     cur_fwd_config.nb_fwd_ports) {
2059                         fwd_stream_stats_display(sm_id);
2060                 } else {
2061                         ports_stats[fs->tx_port].tx_stream = fs;
2062                         ports_stats[fs->rx_port].rx_stream = fs;
2063                 }
2064
2065                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
2066
2067                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
2068                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2069                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2070                                 fs->rx_bad_outer_l4_csum;
2071                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2072                                 fs->rx_bad_outer_ip_csum;
2073
2074                 if (record_core_cycles)
2075                         fwd_cycles += fs->core_cycles;
2076         }
2077         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2078                 pt_id = fwd_ports_ids[i];
2079                 port = &ports[pt_id];
2080
2081                 ret = rte_eth_stats_get(pt_id, &stats);
2082                 if (ret != 0) {
2083                         fprintf(stderr,
2084                                 "%s: Error: failed to get stats (port %u): %d",
2085                                 __func__, pt_id, ret);
2086                         continue;
2087                 }
2088                 stats.ipackets -= port->stats.ipackets;
2089                 stats.opackets -= port->stats.opackets;
2090                 stats.ibytes -= port->stats.ibytes;
2091                 stats.obytes -= port->stats.obytes;
2092                 stats.imissed -= port->stats.imissed;
2093                 stats.oerrors -= port->stats.oerrors;
2094                 stats.rx_nombuf -= port->stats.rx_nombuf;
2095
2096                 total_recv += stats.ipackets;
2097                 total_xmit += stats.opackets;
2098                 total_rx_dropped += stats.imissed;
2099                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2100                 total_tx_dropped += stats.oerrors;
2101                 total_rx_nombuf  += stats.rx_nombuf;
2102
2103                 printf("\n  %s Forward statistics for port %-2d %s\n",
2104                        fwd_stats_border, pt_id, fwd_stats_border);
2105
2106                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2107                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2108                        stats.ipackets + stats.imissed);
2109
2110                 if (cur_fwd_eng == &csum_fwd_engine) {
2111                         printf("  Bad-ipcsum: %-14"PRIu64
2112                                " Bad-l4csum: %-14"PRIu64
2113                                "Bad-outer-l4csum: %-14"PRIu64"\n",
2114                                ports_stats[pt_id].rx_bad_ip_csum,
2115                                ports_stats[pt_id].rx_bad_l4_csum,
2116                                ports_stats[pt_id].rx_bad_outer_l4_csum);
2117                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
2118                                ports_stats[pt_id].rx_bad_outer_ip_csum);
2119                 }
2120                 if (stats.ierrors + stats.rx_nombuf > 0) {
2121                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
2122                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2123                 }
2124
2125                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2126                        "TX-total: %-"PRIu64"\n",
2127                        stats.opackets, ports_stats[pt_id].tx_dropped,
2128                        stats.opackets + ports_stats[pt_id].tx_dropped);
2129
2130                 if (record_burst_stats) {
2131                         if (ports_stats[pt_id].rx_stream)
2132                                 pkt_burst_stats_display("RX",
2133                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
2134                         if (ports_stats[pt_id].tx_stream)
2135                                 pkt_burst_stats_display("TX",
2136                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2137                 }
2138
2139                 printf("  %s--------------------------------%s\n",
2140                        fwd_stats_border, fwd_stats_border);
2141         }
2142
2143         printf("\n  %s Accumulated forward statistics for all ports"
2144                "%s\n",
2145                acc_stats_border, acc_stats_border);
2146         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2147                "%-"PRIu64"\n"
2148                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2149                "%-"PRIu64"\n",
2150                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2151                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2152         if (total_rx_nombuf > 0)
2153                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2154         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
2155                "%s\n",
2156                acc_stats_border, acc_stats_border);
2157         if (record_core_cycles) {
2158 #define CYC_PER_MHZ 1E6
2159                 if (total_recv > 0 || total_xmit > 0) {
2160                         uint64_t total_pkts = 0;
2161                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2162                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2163                                 total_pkts = total_xmit;
2164                         else
2165                                 total_pkts = total_recv;
2166
2167                         printf("\n  CPU cycles/packet=%.2F (total cycles="
2168                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2169                                " MHz Clock\n",
2170                                (double) fwd_cycles / total_pkts,
2171                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2172                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2173                 }
2174         }
2175 }
2176
2177 void
2178 fwd_stats_reset(void)
2179 {
2180         streamid_t sm_id;
2181         portid_t pt_id;
2182         int ret;
2183         int i;
2184
2185         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2186                 pt_id = fwd_ports_ids[i];
2187                 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2188                 if (ret != 0)
2189                         fprintf(stderr,
2190                                 "%s: Error: failed to clear stats (port %u):%d",
2191                                 __func__, pt_id, ret);
2192         }
2193         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2194                 struct fwd_stream *fs = fwd_streams[sm_id];
2195
2196                 fs->rx_packets = 0;
2197                 fs->tx_packets = 0;
2198                 fs->fwd_dropped = 0;
2199                 fs->rx_bad_ip_csum = 0;
2200                 fs->rx_bad_l4_csum = 0;
2201                 fs->rx_bad_outer_l4_csum = 0;
2202                 fs->rx_bad_outer_ip_csum = 0;
2203
2204                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2205                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2206                 fs->core_cycles = 0;
2207         }
2208 }
2209
2210 static void
2211 flush_fwd_rx_queues(void)
2212 {
2213         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2214         portid_t  rxp;
2215         portid_t port_id;
2216         queueid_t rxq;
2217         uint16_t  nb_rx;
2218         uint16_t  i;
2219         uint8_t   j;
2220         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2221         uint64_t timer_period;
2222
2223         if (num_procs > 1) {
2224                 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2225                 return;
2226         }
2227
2228         /* convert to number of cycles */
2229         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2230
2231         for (j = 0; j < 2; j++) {
2232                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2233                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2234                                 port_id = fwd_ports_ids[rxp];
2235
2236                                 /* Polling stopped queues is prohibited. */
2237                                 if (ports[port_id].rxq[rxq].state ==
2238                                     RTE_ETH_QUEUE_STATE_STOPPED)
2239                                         continue;
2240
2241                                 /**
2242                                 * testpmd can stuck in the below do while loop
2243                                 * if rte_eth_rx_burst() always returns nonzero
2244                                 * packets. So timer is added to exit this loop
2245                                 * after 1sec timer expiry.
2246                                 */
2247                                 prev_tsc = rte_rdtsc();
2248                                 do {
2249                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2250                                                 pkts_burst, MAX_PKT_BURST);
2251                                         for (i = 0; i < nb_rx; i++)
2252                                                 rte_pktmbuf_free(pkts_burst[i]);
2253
2254                                         cur_tsc = rte_rdtsc();
2255                                         diff_tsc = cur_tsc - prev_tsc;
2256                                         timer_tsc += diff_tsc;
2257                                 } while ((nb_rx > 0) &&
2258                                         (timer_tsc < timer_period));
2259                                 timer_tsc = 0;
2260                         }
2261                 }
2262                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2263         }
2264 }
2265
2266 static void
2267 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2268 {
2269         struct fwd_stream **fsm;
2270         streamid_t nb_fs;
2271         streamid_t sm_id;
2272 #ifdef RTE_LIB_BITRATESTATS
2273         uint64_t tics_per_1sec;
2274         uint64_t tics_datum;
2275         uint64_t tics_current;
2276         uint16_t i, cnt_ports;
2277
2278         cnt_ports = nb_ports;
2279         tics_datum = rte_rdtsc();
2280         tics_per_1sec = rte_get_timer_hz();
2281 #endif
2282         fsm = &fwd_streams[fc->stream_idx];
2283         nb_fs = fc->stream_nb;
2284         do {
2285                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2286                         if (!fsm[sm_id]->disabled)
2287                                 (*pkt_fwd)(fsm[sm_id]);
2288 #ifdef RTE_LIB_BITRATESTATS
2289                 if (bitrate_enabled != 0 &&
2290                                 bitrate_lcore_id == rte_lcore_id()) {
2291                         tics_current = rte_rdtsc();
2292                         if (tics_current - tics_datum >= tics_per_1sec) {
2293                                 /* Periodic bitrate calculation */
2294                                 for (i = 0; i < cnt_ports; i++)
2295                                         rte_stats_bitrate_calc(bitrate_data,
2296                                                 ports_ids[i]);
2297                                 tics_datum = tics_current;
2298                         }
2299                 }
2300 #endif
2301 #ifdef RTE_LIB_LATENCYSTATS
2302                 if (latencystats_enabled != 0 &&
2303                                 latencystats_lcore_id == rte_lcore_id())
2304                         rte_latencystats_update();
2305 #endif
2306
2307         } while (! fc->stopped);
2308 }
2309
2310 static int
2311 start_pkt_forward_on_core(void *fwd_arg)
2312 {
2313         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2314                              cur_fwd_config.fwd_eng->packet_fwd);
2315         return 0;
2316 }
2317
2318 /*
2319  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2320  * Used to start communication flows in network loopback test configurations.
2321  */
2322 static int
2323 run_one_txonly_burst_on_core(void *fwd_arg)
2324 {
2325         struct fwd_lcore *fwd_lc;
2326         struct fwd_lcore tmp_lcore;
2327
2328         fwd_lc = (struct fwd_lcore *) fwd_arg;
2329         tmp_lcore = *fwd_lc;
2330         tmp_lcore.stopped = 1;
2331         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2332         return 0;
2333 }
2334
2335 /*
2336  * Launch packet forwarding:
2337  *     - Setup per-port forwarding context.
2338  *     - launch logical cores with their forwarding configuration.
2339  */
2340 static void
2341 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2342 {
2343         unsigned int i;
2344         unsigned int lc_id;
2345         int diag;
2346
2347         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2348                 lc_id = fwd_lcores_cpuids[i];
2349                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2350                         fwd_lcores[i]->stopped = 0;
2351                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2352                                                      fwd_lcores[i], lc_id);
2353                         if (diag != 0)
2354                                 fprintf(stderr,
2355                                         "launch lcore %u failed - diag=%d\n",
2356                                         lc_id, diag);
2357                 }
2358         }
2359 }
2360
2361 /*
2362  * Launch packet forwarding configuration.
2363  */
2364 void
2365 start_packet_forwarding(int with_tx_first)
2366 {
2367         port_fwd_begin_t port_fwd_begin;
2368         port_fwd_end_t  port_fwd_end;
2369         stream_init_t stream_init = cur_fwd_eng->stream_init;
2370         unsigned int i;
2371
2372         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2373                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2374
2375         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2376                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2377
2378         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2379                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2380                 (!nb_rxq || !nb_txq))
2381                 rte_exit(EXIT_FAILURE,
2382                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2383                         cur_fwd_eng->fwd_mode_name);
2384
2385         if (all_ports_started() == 0) {
2386                 fprintf(stderr, "Not all ports were started\n");
2387                 return;
2388         }
2389         if (test_done == 0) {
2390                 fprintf(stderr, "Packet forwarding already started\n");
2391                 return;
2392         }
2393
2394         fwd_config_setup();
2395
2396         pkt_fwd_config_display(&cur_fwd_config);
2397         if (!pkt_fwd_shared_rxq_check())
2398                 return;
2399
2400         if (stream_init != NULL)
2401                 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++)
2402                         stream_init(fwd_streams[i]);
2403
2404         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2405         if (port_fwd_begin != NULL) {
2406                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2407                         if (port_fwd_begin(fwd_ports_ids[i])) {
2408                                 fprintf(stderr,
2409                                         "Packet forwarding is not ready\n");
2410                                 return;
2411                         }
2412                 }
2413         }
2414
2415         if (with_tx_first) {
2416                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2417                 if (port_fwd_begin != NULL) {
2418                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2419                                 if (port_fwd_begin(fwd_ports_ids[i])) {
2420                                         fprintf(stderr,
2421                                                 "Packet forwarding is not ready\n");
2422                                         return;
2423                                 }
2424                         }
2425                 }
2426         }
2427
2428         test_done = 0;
2429
2430         if(!no_flush_rx)
2431                 flush_fwd_rx_queues();
2432
2433         rxtx_config_display();
2434
2435         fwd_stats_reset();
2436         if (with_tx_first) {
2437                 while (with_tx_first--) {
2438                         launch_packet_forwarding(
2439                                         run_one_txonly_burst_on_core);
2440                         rte_eal_mp_wait_lcore();
2441                 }
2442                 port_fwd_end = tx_only_engine.port_fwd_end;
2443                 if (port_fwd_end != NULL) {
2444                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2445                                 (*port_fwd_end)(fwd_ports_ids[i]);
2446                 }
2447         }
2448         launch_packet_forwarding(start_pkt_forward_on_core);
2449 }
2450
2451 void
2452 stop_packet_forwarding(void)
2453 {
2454         port_fwd_end_t port_fwd_end;
2455         lcoreid_t lc_id;
2456         portid_t pt_id;
2457         int i;
2458
2459         if (test_done) {
2460                 fprintf(stderr, "Packet forwarding not started\n");
2461                 return;
2462         }
2463         printf("Telling cores to stop...");
2464         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2465                 fwd_lcores[lc_id]->stopped = 1;
2466         printf("\nWaiting for lcores to finish...\n");
2467         rte_eal_mp_wait_lcore();
2468         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2469         if (port_fwd_end != NULL) {
2470                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2471                         pt_id = fwd_ports_ids[i];
2472                         (*port_fwd_end)(pt_id);
2473                 }
2474         }
2475
2476         fwd_stats_display();
2477
2478         printf("\nDone.\n");
2479         test_done = 1;
2480 }
2481
2482 void
2483 dev_set_link_up(portid_t pid)
2484 {
2485         if (rte_eth_dev_set_link_up(pid) < 0)
2486                 fprintf(stderr, "\nSet link up fail.\n");
2487 }
2488
2489 void
2490 dev_set_link_down(portid_t pid)
2491 {
2492         if (rte_eth_dev_set_link_down(pid) < 0)
2493                 fprintf(stderr, "\nSet link down fail.\n");
2494 }
2495
2496 static int
2497 all_ports_started(void)
2498 {
2499         portid_t pi;
2500         struct rte_port *port;
2501
2502         RTE_ETH_FOREACH_DEV(pi) {
2503                 port = &ports[pi];
2504                 /* Check if there is a port which is not started */
2505                 if ((port->port_status != RTE_PORT_STARTED) &&
2506                         (port->slave_flag == 0))
2507                         return 0;
2508         }
2509
2510         /* No port is not started */
2511         return 1;
2512 }
2513
2514 int
2515 port_is_stopped(portid_t port_id)
2516 {
2517         struct rte_port *port = &ports[port_id];
2518
2519         if ((port->port_status != RTE_PORT_STOPPED) &&
2520             (port->slave_flag == 0))
2521                 return 0;
2522         return 1;
2523 }
2524
2525 int
2526 all_ports_stopped(void)
2527 {
2528         portid_t pi;
2529
2530         RTE_ETH_FOREACH_DEV(pi) {
2531                 if (!port_is_stopped(pi))
2532                         return 0;
2533         }
2534
2535         return 1;
2536 }
2537
2538 int
2539 port_is_started(portid_t port_id)
2540 {
2541         if (port_id_is_invalid(port_id, ENABLED_WARN))
2542                 return 0;
2543
2544         if (ports[port_id].port_status != RTE_PORT_STARTED)
2545                 return 0;
2546
2547         return 1;
2548 }
2549
2550 /* Configure the Rx and Tx hairpin queues for the selected port. */
2551 static int
2552 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2553 {
2554         queueid_t qi;
2555         struct rte_eth_hairpin_conf hairpin_conf = {
2556                 .peer_count = 1,
2557         };
2558         int i;
2559         int diag;
2560         struct rte_port *port = &ports[pi];
2561         uint16_t peer_rx_port = pi;
2562         uint16_t peer_tx_port = pi;
2563         uint32_t manual = 1;
2564         uint32_t tx_exp = hairpin_mode & 0x10;
2565
2566         if (!(hairpin_mode & 0xf)) {
2567                 peer_rx_port = pi;
2568                 peer_tx_port = pi;
2569                 manual = 0;
2570         } else if (hairpin_mode & 0x1) {
2571                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2572                                                        RTE_ETH_DEV_NO_OWNER);
2573                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2574                         peer_tx_port = rte_eth_find_next_owned_by(0,
2575                                                 RTE_ETH_DEV_NO_OWNER);
2576                 if (p_pi != RTE_MAX_ETHPORTS) {
2577                         peer_rx_port = p_pi;
2578                 } else {
2579                         uint16_t next_pi;
2580
2581                         /* Last port will be the peer RX port of the first. */
2582                         RTE_ETH_FOREACH_DEV(next_pi)
2583                                 peer_rx_port = next_pi;
2584                 }
2585                 manual = 1;
2586         } else if (hairpin_mode & 0x2) {
2587                 if (cnt_pi & 0x1) {
2588                         peer_rx_port = p_pi;
2589                 } else {
2590                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2591                                                 RTE_ETH_DEV_NO_OWNER);
2592                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2593                                 peer_rx_port = pi;
2594                 }
2595                 peer_tx_port = peer_rx_port;
2596                 manual = 1;
2597         }
2598
2599         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2600                 hairpin_conf.peers[0].port = peer_rx_port;
2601                 hairpin_conf.peers[0].queue = i + nb_rxq;
2602                 hairpin_conf.manual_bind = !!manual;
2603                 hairpin_conf.tx_explicit = !!tx_exp;
2604                 diag = rte_eth_tx_hairpin_queue_setup
2605                         (pi, qi, nb_txd, &hairpin_conf);
2606                 i++;
2607                 if (diag == 0)
2608                         continue;
2609
2610                 /* Fail to setup rx queue, return */
2611                 if (port->port_status == RTE_PORT_HANDLING)
2612                         port->port_status = RTE_PORT_STOPPED;
2613                 else
2614                         fprintf(stderr,
2615                                 "Port %d can not be set back to stopped\n", pi);
2616                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2617                         pi);
2618                 /* try to reconfigure queues next time */
2619                 port->need_reconfig_queues = 1;
2620                 return -1;
2621         }
2622         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2623                 hairpin_conf.peers[0].port = peer_tx_port;
2624                 hairpin_conf.peers[0].queue = i + nb_txq;
2625                 hairpin_conf.manual_bind = !!manual;
2626                 hairpin_conf.tx_explicit = !!tx_exp;
2627                 diag = rte_eth_rx_hairpin_queue_setup
2628                         (pi, qi, nb_rxd, &hairpin_conf);
2629                 i++;
2630                 if (diag == 0)
2631                         continue;
2632
2633                 /* Fail to setup rx queue, return */
2634                 if (port->port_status == RTE_PORT_HANDLING)
2635                         port->port_status = RTE_PORT_STOPPED;
2636                 else
2637                         fprintf(stderr,
2638                                 "Port %d can not be set back to stopped\n", pi);
2639                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2640                         pi);
2641                 /* try to reconfigure queues next time */
2642                 port->need_reconfig_queues = 1;
2643                 return -1;
2644         }
2645         return 0;
2646 }
2647
2648 /* Configure the Rx with optional split. */
2649 int
2650 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2651                uint16_t nb_rx_desc, unsigned int socket_id,
2652                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2653 {
2654         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2655         unsigned int i, mp_n;
2656         int ret;
2657
2658         if (rx_pkt_nb_segs <= 1 ||
2659             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2660                 rx_conf->rx_seg = NULL;
2661                 rx_conf->rx_nseg = 0;
2662                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2663                                              nb_rx_desc, socket_id,
2664                                              rx_conf, mp);
2665                 goto exit;
2666         }
2667         for (i = 0; i < rx_pkt_nb_segs; i++) {
2668                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2669                 struct rte_mempool *mpx;
2670                 /*
2671                  * Use last valid pool for the segments with number
2672                  * exceeding the pool index.
2673                  */
2674                 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2675                 mpx = mbuf_pool_find(socket_id, mp_n);
2676                 /* Handle zero as mbuf data buffer size. */
2677                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2678                                    rx_pkt_seg_lengths[i] :
2679                                    mbuf_data_size[mp_n];
2680                 rx_seg->offset = i < rx_pkt_nb_offs ?
2681                                    rx_pkt_seg_offsets[i] : 0;
2682                 rx_seg->mp = mpx ? mpx : mp;
2683         }
2684         rx_conf->rx_nseg = rx_pkt_nb_segs;
2685         rx_conf->rx_seg = rx_useg;
2686         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2687                                     socket_id, rx_conf, NULL);
2688         rx_conf->rx_seg = NULL;
2689         rx_conf->rx_nseg = 0;
2690 exit:
2691         ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ?
2692                                                 RTE_ETH_QUEUE_STATE_STOPPED :
2693                                                 RTE_ETH_QUEUE_STATE_STARTED;
2694         return ret;
2695 }
2696
2697 static int
2698 alloc_xstats_display_info(portid_t pi)
2699 {
2700         uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2701         uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2702         uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2703
2704         if (xstats_display_num == 0)
2705                 return 0;
2706
2707         *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2708         if (*ids_supp == NULL)
2709                 goto fail_ids_supp;
2710
2711         *prev_values = calloc(xstats_display_num,
2712                               sizeof(**prev_values));
2713         if (*prev_values == NULL)
2714                 goto fail_prev_values;
2715
2716         *curr_values = calloc(xstats_display_num,
2717                               sizeof(**curr_values));
2718         if (*curr_values == NULL)
2719                 goto fail_curr_values;
2720
2721         ports[pi].xstats_info.allocated = true;
2722
2723         return 0;
2724
2725 fail_curr_values:
2726         free(*prev_values);
2727 fail_prev_values:
2728         free(*ids_supp);
2729 fail_ids_supp:
2730         return -ENOMEM;
2731 }
2732
2733 static void
2734 free_xstats_display_info(portid_t pi)
2735 {
2736         if (!ports[pi].xstats_info.allocated)
2737                 return;
2738         free(ports[pi].xstats_info.ids_supp);
2739         free(ports[pi].xstats_info.prev_values);
2740         free(ports[pi].xstats_info.curr_values);
2741         ports[pi].xstats_info.allocated = false;
2742 }
2743
2744 /** Fill helper structures for specified port to show extended statistics. */
2745 static void
2746 fill_xstats_display_info_for_port(portid_t pi)
2747 {
2748         unsigned int stat, stat_supp;
2749         const char *xstat_name;
2750         struct rte_port *port;
2751         uint64_t *ids_supp;
2752         int rc;
2753
2754         if (xstats_display_num == 0)
2755                 return;
2756
2757         if (pi == (portid_t)RTE_PORT_ALL) {
2758                 fill_xstats_display_info();
2759                 return;
2760         }
2761
2762         port = &ports[pi];
2763         if (port->port_status != RTE_PORT_STARTED)
2764                 return;
2765
2766         if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2767                 rte_exit(EXIT_FAILURE,
2768                          "Failed to allocate xstats display memory\n");
2769
2770         ids_supp = port->xstats_info.ids_supp;
2771         for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2772                 xstat_name = xstats_display[stat].name;
2773                 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2774                                                    ids_supp + stat_supp);
2775                 if (rc != 0) {
2776                         fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2777                                 xstat_name, pi, stat);
2778                         continue;
2779                 }
2780                 stat_supp++;
2781         }
2782
2783         port->xstats_info.ids_supp_sz = stat_supp;
2784 }
2785
2786 /** Fill helper structures for all ports to show extended statistics. */
2787 static void
2788 fill_xstats_display_info(void)
2789 {
2790         portid_t pi;
2791
2792         if (xstats_display_num == 0)
2793                 return;
2794
2795         RTE_ETH_FOREACH_DEV(pi)
2796                 fill_xstats_display_info_for_port(pi);
2797 }
2798
2799 int
2800 start_port(portid_t pid)
2801 {
2802         int diag, need_check_link_status = -1;
2803         portid_t pi;
2804         portid_t p_pi = RTE_MAX_ETHPORTS;
2805         portid_t pl[RTE_MAX_ETHPORTS];
2806         portid_t peer_pl[RTE_MAX_ETHPORTS];
2807         uint16_t cnt_pi = 0;
2808         uint16_t cfg_pi = 0;
2809         int peer_pi;
2810         queueid_t qi;
2811         struct rte_port *port;
2812         struct rte_eth_hairpin_cap cap;
2813
2814         if (port_id_is_invalid(pid, ENABLED_WARN))
2815                 return 0;
2816
2817         RTE_ETH_FOREACH_DEV(pi) {
2818                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2819                         continue;
2820
2821                 if (port_is_bonding_slave(pi)) {
2822                         fprintf(stderr,
2823                                 "Please remove port %d from bonded device.\n",
2824                                 pi);
2825                         continue;
2826                 }
2827
2828                 need_check_link_status = 0;
2829                 port = &ports[pi];
2830                 if (port->port_status == RTE_PORT_STOPPED)
2831                         port->port_status = RTE_PORT_HANDLING;
2832                 else {
2833                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2834                         continue;
2835                 }
2836
2837                 if (port->need_reconfig > 0) {
2838                         struct rte_eth_conf dev_conf;
2839                         int k;
2840
2841                         port->need_reconfig = 0;
2842
2843                         if (flow_isolate_all) {
2844                                 int ret = port_flow_isolate(pi, 1);
2845                                 if (ret) {
2846                                         fprintf(stderr,
2847                                                 "Failed to apply isolated mode on port %d\n",
2848                                                 pi);
2849                                         return -1;
2850                                 }
2851                         }
2852                         configure_rxtx_dump_callbacks(0);
2853                         printf("Configuring Port %d (socket %u)\n", pi,
2854                                         port->socket_id);
2855                         if (nb_hairpinq > 0 &&
2856                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2857                                 fprintf(stderr,
2858                                         "Port %d doesn't support hairpin queues\n",
2859                                         pi);
2860                                 return -1;
2861                         }
2862
2863                         /* configure port */
2864                         diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2865                                                      nb_txq + nb_hairpinq,
2866                                                      &(port->dev_conf));
2867                         if (diag != 0) {
2868                                 if (port->port_status == RTE_PORT_HANDLING)
2869                                         port->port_status = RTE_PORT_STOPPED;
2870                                 else
2871                                         fprintf(stderr,
2872                                                 "Port %d can not be set back to stopped\n",
2873                                                 pi);
2874                                 fprintf(stderr, "Fail to configure port %d\n",
2875                                         pi);
2876                                 /* try to reconfigure port next time */
2877                                 port->need_reconfig = 1;
2878                                 return -1;
2879                         }
2880                         /* get device configuration*/
2881                         if (0 !=
2882                                 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2883                                 fprintf(stderr,
2884                                         "port %d can not get device configuration\n",
2885                                         pi);
2886                                 return -1;
2887                         }
2888                         /* Apply Rx offloads configuration */
2889                         if (dev_conf.rxmode.offloads !=
2890                             port->dev_conf.rxmode.offloads) {
2891                                 port->dev_conf.rxmode.offloads |=
2892                                         dev_conf.rxmode.offloads;
2893                                 for (k = 0;
2894                                      k < port->dev_info.max_rx_queues;
2895                                      k++)
2896                                         port->rxq[k].conf.offloads |=
2897                                                 dev_conf.rxmode.offloads;
2898                         }
2899                         /* Apply Tx offloads configuration */
2900                         if (dev_conf.txmode.offloads !=
2901                             port->dev_conf.txmode.offloads) {
2902                                 port->dev_conf.txmode.offloads |=
2903                                         dev_conf.txmode.offloads;
2904                                 for (k = 0;
2905                                      k < port->dev_info.max_tx_queues;
2906                                      k++)
2907                                         port->txq[k].conf.offloads |=
2908                                                 dev_conf.txmode.offloads;
2909                         }
2910                 }
2911                 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2912                         port->need_reconfig_queues = 0;
2913                         /* setup tx queues */
2914                         for (qi = 0; qi < nb_txq; qi++) {
2915                                 struct rte_eth_txconf *conf =
2916                                                         &port->txq[qi].conf;
2917
2918                                 if ((numa_support) &&
2919                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2920                                         diag = rte_eth_tx_queue_setup(pi, qi,
2921                                                 port->nb_tx_desc[qi],
2922                                                 txring_numa[pi],
2923                                                 &(port->txq[qi].conf));
2924                                 else
2925                                         diag = rte_eth_tx_queue_setup(pi, qi,
2926                                                 port->nb_tx_desc[qi],
2927                                                 port->socket_id,
2928                                                 &(port->txq[qi].conf));
2929
2930                                 if (diag == 0) {
2931                                         port->txq[qi].state =
2932                                                 conf->tx_deferred_start ?
2933                                                 RTE_ETH_QUEUE_STATE_STOPPED :
2934                                                 RTE_ETH_QUEUE_STATE_STARTED;
2935                                         continue;
2936                                 }
2937
2938                                 /* Fail to setup tx queue, return */
2939                                 if (port->port_status == RTE_PORT_HANDLING)
2940                                         port->port_status = RTE_PORT_STOPPED;
2941                                 else
2942                                         fprintf(stderr,
2943                                                 "Port %d can not be set back to stopped\n",
2944                                                 pi);
2945                                 fprintf(stderr,
2946                                         "Fail to configure port %d tx queues\n",
2947                                         pi);
2948                                 /* try to reconfigure queues next time */
2949                                 port->need_reconfig_queues = 1;
2950                                 return -1;
2951                         }
2952                         for (qi = 0; qi < nb_rxq; qi++) {
2953                                 /* setup rx queues */
2954                                 if ((numa_support) &&
2955                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2956                                         struct rte_mempool * mp =
2957                                                 mbuf_pool_find
2958                                                         (rxring_numa[pi], 0);
2959                                         if (mp == NULL) {
2960                                                 fprintf(stderr,
2961                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2962                                                         rxring_numa[pi]);
2963                                                 return -1;
2964                                         }
2965
2966                                         diag = rx_queue_setup(pi, qi,
2967                                              port->nb_rx_desc[qi],
2968                                              rxring_numa[pi],
2969                                              &(port->rxq[qi].conf),
2970                                              mp);
2971                                 } else {
2972                                         struct rte_mempool *mp =
2973                                                 mbuf_pool_find
2974                                                         (port->socket_id, 0);
2975                                         if (mp == NULL) {
2976                                                 fprintf(stderr,
2977                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2978                                                         port->socket_id);
2979                                                 return -1;
2980                                         }
2981                                         diag = rx_queue_setup(pi, qi,
2982                                              port->nb_rx_desc[qi],
2983                                              port->socket_id,
2984                                              &(port->rxq[qi].conf),
2985                                              mp);
2986                                 }
2987                                 if (diag == 0)
2988                                         continue;
2989
2990                                 /* Fail to setup rx queue, return */
2991                                 if (port->port_status == RTE_PORT_HANDLING)
2992                                         port->port_status = RTE_PORT_STOPPED;
2993                                 else
2994                                         fprintf(stderr,
2995                                                 "Port %d can not be set back to stopped\n",
2996                                                 pi);
2997                                 fprintf(stderr,
2998                                         "Fail to configure port %d rx queues\n",
2999                                         pi);
3000                                 /* try to reconfigure queues next time */
3001                                 port->need_reconfig_queues = 1;
3002                                 return -1;
3003                         }
3004                         /* setup hairpin queues */
3005                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
3006                                 return -1;
3007                 }
3008                 configure_rxtx_dump_callbacks(verbose_level);
3009                 if (clear_ptypes) {
3010                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
3011                                         NULL, 0);
3012                         if (diag < 0)
3013                                 fprintf(stderr,
3014                                         "Port %d: Failed to disable Ptype parsing\n",
3015                                         pi);
3016                 }
3017
3018                 p_pi = pi;
3019                 cnt_pi++;
3020
3021                 /* start port */
3022                 diag = eth_dev_start_mp(pi);
3023                 if (diag < 0) {
3024                         fprintf(stderr, "Fail to start port %d: %s\n",
3025                                 pi, rte_strerror(-diag));
3026
3027                         /* Fail to setup rx queue, return */
3028                         if (port->port_status == RTE_PORT_HANDLING)
3029                                 port->port_status = RTE_PORT_STOPPED;
3030                         else
3031                                 fprintf(stderr,
3032                                         "Port %d can not be set back to stopped\n",
3033                                         pi);
3034                         continue;
3035                 }
3036
3037                 if (port->port_status == RTE_PORT_HANDLING)
3038                         port->port_status = RTE_PORT_STARTED;
3039                 else
3040                         fprintf(stderr, "Port %d can not be set into started\n",
3041                                 pi);
3042
3043                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
3044                         printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
3045                                         RTE_ETHER_ADDR_BYTES(&port->eth_addr));
3046
3047                 /* at least one port started, need checking link status */
3048                 need_check_link_status = 1;
3049
3050                 pl[cfg_pi++] = pi;
3051         }
3052
3053         if (need_check_link_status == 1 && !no_link_check)
3054                 check_all_ports_link_status(RTE_PORT_ALL);
3055         else if (need_check_link_status == 0)
3056                 fprintf(stderr, "Please stop the ports first\n");
3057
3058         if (hairpin_mode & 0xf) {
3059                 uint16_t i;
3060                 int j;
3061
3062                 /* bind all started hairpin ports */
3063                 for (i = 0; i < cfg_pi; i++) {
3064                         pi = pl[i];
3065                         /* bind current Tx to all peer Rx */
3066                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3067                                                         RTE_MAX_ETHPORTS, 1);
3068                         if (peer_pi < 0)
3069                                 return peer_pi;
3070                         for (j = 0; j < peer_pi; j++) {
3071                                 if (!port_is_started(peer_pl[j]))
3072                                         continue;
3073                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
3074                                 if (diag < 0) {
3075                                         fprintf(stderr,
3076                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3077                                                 pi, peer_pl[j],
3078                                                 rte_strerror(-diag));
3079                                         return -1;
3080                                 }
3081                         }
3082                         /* bind all peer Tx to current Rx */
3083                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3084                                                         RTE_MAX_ETHPORTS, 0);
3085                         if (peer_pi < 0)
3086                                 return peer_pi;
3087                         for (j = 0; j < peer_pi; j++) {
3088                                 if (!port_is_started(peer_pl[j]))
3089                                         continue;
3090                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
3091                                 if (diag < 0) {
3092                                         fprintf(stderr,
3093                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3094                                                 peer_pl[j], pi,
3095                                                 rte_strerror(-diag));
3096                                         return -1;
3097                                 }
3098                         }
3099                 }
3100         }
3101
3102         fill_xstats_display_info_for_port(pid);
3103
3104         printf("Done\n");
3105         return 0;
3106 }
3107
3108 void
3109 stop_port(portid_t pid)
3110 {
3111         portid_t pi;
3112         struct rte_port *port;
3113         int need_check_link_status = 0;
3114         portid_t peer_pl[RTE_MAX_ETHPORTS];
3115         int peer_pi;
3116
3117         if (port_id_is_invalid(pid, ENABLED_WARN))
3118                 return;
3119
3120         printf("Stopping ports...\n");
3121
3122         RTE_ETH_FOREACH_DEV(pi) {
3123                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3124                         continue;
3125
3126                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3127                         fprintf(stderr,
3128                                 "Please remove port %d from forwarding configuration.\n",
3129                                 pi);
3130                         continue;
3131                 }
3132
3133                 if (port_is_bonding_slave(pi)) {
3134                         fprintf(stderr,
3135                                 "Please remove port %d from bonded device.\n",
3136                                 pi);
3137                         continue;
3138                 }
3139
3140                 port = &ports[pi];
3141                 if (port->port_status == RTE_PORT_STARTED)
3142                         port->port_status = RTE_PORT_HANDLING;
3143                 else
3144                         continue;
3145
3146                 if (hairpin_mode & 0xf) {
3147                         int j;
3148
3149                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3150                         /* unbind all peer Tx from current Rx */
3151                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3152                                                         RTE_MAX_ETHPORTS, 0);
3153                         if (peer_pi < 0)
3154                                 continue;
3155                         for (j = 0; j < peer_pi; j++) {
3156                                 if (!port_is_started(peer_pl[j]))
3157                                         continue;
3158                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
3159                         }
3160                 }
3161
3162                 if (port->flow_list)
3163                         port_flow_flush(pi);
3164
3165                 if (eth_dev_stop_mp(pi) != 0)
3166                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3167                                 pi);
3168
3169                 if (port->port_status == RTE_PORT_HANDLING)
3170                         port->port_status = RTE_PORT_STOPPED;
3171                 else
3172                         fprintf(stderr, "Port %d can not be set into stopped\n",
3173                                 pi);
3174                 need_check_link_status = 1;
3175         }
3176         if (need_check_link_status && !no_link_check)
3177                 check_all_ports_link_status(RTE_PORT_ALL);
3178
3179         printf("Done\n");
3180 }
3181
3182 static void
3183 remove_invalid_ports_in(portid_t *array, portid_t *total)
3184 {
3185         portid_t i;
3186         portid_t new_total = 0;
3187
3188         for (i = 0; i < *total; i++)
3189                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3190                         array[new_total] = array[i];
3191                         new_total++;
3192                 }
3193         *total = new_total;
3194 }
3195
3196 static void
3197 remove_invalid_ports(void)
3198 {
3199         remove_invalid_ports_in(ports_ids, &nb_ports);
3200         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3201         nb_cfg_ports = nb_fwd_ports;
3202 }
3203
3204 void
3205 close_port(portid_t pid)
3206 {
3207         portid_t pi;
3208         struct rte_port *port;
3209
3210         if (port_id_is_invalid(pid, ENABLED_WARN))
3211                 return;
3212
3213         printf("Closing ports...\n");
3214
3215         RTE_ETH_FOREACH_DEV(pi) {
3216                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3217                         continue;
3218
3219                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3220                         fprintf(stderr,
3221                                 "Please remove port %d from forwarding configuration.\n",
3222                                 pi);
3223                         continue;
3224                 }
3225
3226                 if (port_is_bonding_slave(pi)) {
3227                         fprintf(stderr,
3228                                 "Please remove port %d from bonded device.\n",
3229                                 pi);
3230                         continue;
3231                 }
3232
3233                 port = &ports[pi];
3234                 if (port->port_status == RTE_PORT_CLOSED) {
3235                         fprintf(stderr, "Port %d is already closed\n", pi);
3236                         continue;
3237                 }
3238
3239                 if (is_proc_primary()) {
3240                         mcast_addr_pool_destroy(pi);
3241                         port_flow_flush(pi);
3242                         port_flex_item_flush(pi);
3243                         port_action_handle_flush(pi);
3244                         rte_eth_dev_close(pi);
3245                 }
3246
3247                 free_xstats_display_info(pi);
3248         }
3249
3250         remove_invalid_ports();
3251         printf("Done\n");
3252 }
3253
3254 void
3255 reset_port(portid_t pid)
3256 {
3257         int diag;
3258         portid_t pi;
3259         struct rte_port *port;
3260
3261         if (port_id_is_invalid(pid, ENABLED_WARN))
3262                 return;
3263
3264         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3265                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3266                 fprintf(stderr,
3267                         "Can not reset port(s), please stop port(s) first.\n");
3268                 return;
3269         }
3270
3271         printf("Resetting ports...\n");
3272
3273         RTE_ETH_FOREACH_DEV(pi) {
3274                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3275                         continue;
3276
3277                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3278                         fprintf(stderr,
3279                                 "Please remove port %d from forwarding configuration.\n",
3280                                 pi);
3281                         continue;
3282                 }
3283
3284                 if (port_is_bonding_slave(pi)) {
3285                         fprintf(stderr,
3286                                 "Please remove port %d from bonded device.\n",
3287                                 pi);
3288                         continue;
3289                 }
3290
3291                 diag = rte_eth_dev_reset(pi);
3292                 if (diag == 0) {
3293                         port = &ports[pi];
3294                         port->need_reconfig = 1;
3295                         port->need_reconfig_queues = 1;
3296                 } else {
3297                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3298                                 pi, diag);
3299                 }
3300         }
3301
3302         printf("Done\n");
3303 }
3304
3305 void
3306 attach_port(char *identifier)
3307 {
3308         portid_t pi;
3309         struct rte_dev_iterator iterator;
3310
3311         printf("Attaching a new port...\n");
3312
3313         if (identifier == NULL) {
3314                 fprintf(stderr, "Invalid parameters are specified\n");
3315                 return;
3316         }
3317
3318         if (rte_dev_probe(identifier) < 0) {
3319                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3320                 return;
3321         }
3322
3323         /* first attach mode: event */
3324         if (setup_on_probe_event) {
3325                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3326                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3327                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
3328                                         ports[pi].need_setup != 0)
3329                                 setup_attached_port(pi);
3330                 return;
3331         }
3332
3333         /* second attach mode: iterator */
3334         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3335                 /* setup ports matching the devargs used for probing */
3336                 if (port_is_forwarding(pi))
3337                         continue; /* port was already attached before */
3338                 setup_attached_port(pi);
3339         }
3340 }
3341
3342 static void
3343 setup_attached_port(portid_t pi)
3344 {
3345         unsigned int socket_id;
3346         int ret;
3347
3348         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3349         /* if socket_id is invalid, set to the first available socket. */
3350         if (check_socket_id(socket_id) < 0)
3351                 socket_id = socket_ids[0];
3352         reconfig(pi, socket_id);
3353         ret = rte_eth_promiscuous_enable(pi);
3354         if (ret != 0)
3355                 fprintf(stderr,
3356                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3357                         pi, rte_strerror(-ret));
3358
3359         ports_ids[nb_ports++] = pi;
3360         fwd_ports_ids[nb_fwd_ports++] = pi;
3361         nb_cfg_ports = nb_fwd_ports;
3362         ports[pi].need_setup = 0;
3363         ports[pi].port_status = RTE_PORT_STOPPED;
3364
3365         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3366         printf("Done\n");
3367 }
3368
3369 static void
3370 detach_device(struct rte_device *dev)
3371 {
3372         portid_t sibling;
3373
3374         if (dev == NULL) {
3375                 fprintf(stderr, "Device already removed\n");
3376                 return;
3377         }
3378
3379         printf("Removing a device...\n");
3380
3381         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3382                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3383                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3384                                 fprintf(stderr, "Port %u not stopped\n",
3385                                         sibling);
3386                                 return;
3387                         }
3388                         port_flow_flush(sibling);
3389                 }
3390         }
3391
3392         if (rte_dev_remove(dev) < 0) {
3393                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3394                 return;
3395         }
3396         remove_invalid_ports();
3397
3398         printf("Device is detached\n");
3399         printf("Now total ports is %d\n", nb_ports);
3400         printf("Done\n");
3401         return;
3402 }
3403
3404 void
3405 detach_port_device(portid_t port_id)
3406 {
3407         int ret;
3408         struct rte_eth_dev_info dev_info;
3409
3410         if (port_id_is_invalid(port_id, ENABLED_WARN))
3411                 return;
3412
3413         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3414                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3415                         fprintf(stderr, "Port not stopped\n");
3416                         return;
3417                 }
3418                 fprintf(stderr, "Port was not closed\n");
3419         }
3420
3421         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3422         if (ret != 0) {
3423                 TESTPMD_LOG(ERR,
3424                         "Failed to get device info for port %d, not detaching\n",
3425                         port_id);
3426                 return;
3427         }
3428         detach_device(dev_info.device);
3429 }
3430
3431 void
3432 detach_devargs(char *identifier)
3433 {
3434         struct rte_dev_iterator iterator;
3435         struct rte_devargs da;
3436         portid_t port_id;
3437
3438         printf("Removing a device...\n");
3439
3440         memset(&da, 0, sizeof(da));
3441         if (rte_devargs_parsef(&da, "%s", identifier)) {
3442                 fprintf(stderr, "cannot parse identifier\n");
3443                 return;
3444         }
3445
3446         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3447                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3448                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3449                                 fprintf(stderr, "Port %u not stopped\n",
3450                                         port_id);
3451                                 rte_eth_iterator_cleanup(&iterator);
3452                                 rte_devargs_reset(&da);
3453                                 return;
3454                         }
3455                         port_flow_flush(port_id);
3456                 }
3457         }
3458
3459         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3460                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3461                             da.name, da.bus->name);
3462                 rte_devargs_reset(&da);
3463                 return;
3464         }
3465
3466         remove_invalid_ports();
3467
3468         printf("Device %s is detached\n", identifier);
3469         printf("Now total ports is %d\n", nb_ports);
3470         printf("Done\n");
3471         rte_devargs_reset(&da);
3472 }
3473
3474 void
3475 pmd_test_exit(void)
3476 {
3477         portid_t pt_id;
3478         unsigned int i;
3479         int ret;
3480
3481         if (test_done == 0)
3482                 stop_packet_forwarding();
3483
3484 #ifndef RTE_EXEC_ENV_WINDOWS
3485         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3486                 if (mempools[i]) {
3487                         if (mp_alloc_type == MP_ALLOC_ANON)
3488                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3489                                                      NULL);
3490                 }
3491         }
3492 #endif
3493         if (ports != NULL) {
3494                 no_link_check = 1;
3495                 RTE_ETH_FOREACH_DEV(pt_id) {
3496                         printf("\nStopping port %d...\n", pt_id);
3497                         fflush(stdout);
3498                         stop_port(pt_id);
3499                 }
3500                 RTE_ETH_FOREACH_DEV(pt_id) {
3501                         printf("\nShutting down port %d...\n", pt_id);
3502                         fflush(stdout);
3503                         close_port(pt_id);
3504                 }
3505         }
3506
3507         if (hot_plug) {
3508                 ret = rte_dev_event_monitor_stop();
3509                 if (ret) {
3510                         RTE_LOG(ERR, EAL,
3511                                 "fail to stop device event monitor.");
3512                         return;
3513                 }
3514
3515                 ret = rte_dev_event_callback_unregister(NULL,
3516                         dev_event_callback, NULL);
3517                 if (ret < 0) {
3518                         RTE_LOG(ERR, EAL,
3519                                 "fail to unregister device event callback.\n");
3520                         return;
3521                 }
3522
3523                 ret = rte_dev_hotplug_handle_disable();
3524                 if (ret) {
3525                         RTE_LOG(ERR, EAL,
3526                                 "fail to disable hotplug handling.\n");
3527                         return;
3528                 }
3529         }
3530         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3531                 if (mempools[i])
3532                         mempool_free_mp(mempools[i]);
3533         }
3534         free(xstats_display);
3535
3536         printf("\nBye...\n");
3537 }
3538
3539 typedef void (*cmd_func_t)(void);
3540 struct pmd_test_command {
3541         const char *cmd_name;
3542         cmd_func_t cmd_func;
3543 };
3544
3545 /* Check the link status of all ports in up to 9s, and print them finally */
3546 static void
3547 check_all_ports_link_status(uint32_t port_mask)
3548 {
3549 #define CHECK_INTERVAL 100 /* 100ms */
3550 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3551         portid_t portid;
3552         uint8_t count, all_ports_up, print_flag = 0;
3553         struct rte_eth_link link;
3554         int ret;
3555         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3556
3557         printf("Checking link statuses...\n");
3558         fflush(stdout);
3559         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3560                 all_ports_up = 1;
3561                 RTE_ETH_FOREACH_DEV(portid) {
3562                         if ((port_mask & (1 << portid)) == 0)
3563                                 continue;
3564                         memset(&link, 0, sizeof(link));
3565                         ret = rte_eth_link_get_nowait(portid, &link);
3566                         if (ret < 0) {
3567                                 all_ports_up = 0;
3568                                 if (print_flag == 1)
3569                                         fprintf(stderr,
3570                                                 "Port %u link get failed: %s\n",
3571                                                 portid, rte_strerror(-ret));
3572                                 continue;
3573                         }
3574                         /* print link status if flag set */
3575                         if (print_flag == 1) {
3576                                 rte_eth_link_to_str(link_status,
3577                                         sizeof(link_status), &link);
3578                                 printf("Port %d %s\n", portid, link_status);
3579                                 continue;
3580                         }
3581                         /* clear all_ports_up flag if any link down */
3582                         if (link.link_status == RTE_ETH_LINK_DOWN) {
3583                                 all_ports_up = 0;
3584                                 break;
3585                         }
3586                 }
3587                 /* after finally printing all link status, get out */
3588                 if (print_flag == 1)
3589                         break;
3590
3591                 if (all_ports_up == 0) {
3592                         fflush(stdout);
3593                         rte_delay_ms(CHECK_INTERVAL);
3594                 }
3595
3596                 /* set the print_flag if all ports up or timeout */
3597                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3598                         print_flag = 1;
3599                 }
3600
3601                 if (lsc_interrupt)
3602                         break;
3603         }
3604 }
3605
3606 static void
3607 rmv_port_callback(void *arg)
3608 {
3609         int need_to_start = 0;
3610         int org_no_link_check = no_link_check;
3611         portid_t port_id = (intptr_t)arg;
3612         struct rte_eth_dev_info dev_info;
3613         int ret;
3614
3615         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3616
3617         if (!test_done && port_is_forwarding(port_id)) {
3618                 need_to_start = 1;
3619                 stop_packet_forwarding();
3620         }
3621         no_link_check = 1;
3622         stop_port(port_id);
3623         no_link_check = org_no_link_check;
3624
3625         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3626         if (ret != 0)
3627                 TESTPMD_LOG(ERR,
3628                         "Failed to get device info for port %d, not detaching\n",
3629                         port_id);
3630         else {
3631                 struct rte_device *device = dev_info.device;
3632                 close_port(port_id);
3633                 detach_device(device); /* might be already removed or have more ports */
3634         }
3635         if (need_to_start)
3636                 start_packet_forwarding(0);
3637 }
3638
3639 /* This function is used by the interrupt thread */
3640 static int
3641 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3642                   void *ret_param)
3643 {
3644         RTE_SET_USED(param);
3645         RTE_SET_USED(ret_param);
3646
3647         if (type >= RTE_ETH_EVENT_MAX) {
3648                 fprintf(stderr,
3649                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3650                         port_id, __func__, type);
3651                 fflush(stderr);
3652         } else if (event_print_mask & (UINT32_C(1) << type)) {
3653                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3654                         eth_event_desc[type]);
3655                 fflush(stdout);
3656         }
3657
3658         switch (type) {
3659         case RTE_ETH_EVENT_NEW:
3660                 ports[port_id].need_setup = 1;
3661                 ports[port_id].port_status = RTE_PORT_HANDLING;
3662                 break;
3663         case RTE_ETH_EVENT_INTR_RMV:
3664                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3665                         break;
3666                 if (rte_eal_alarm_set(100000,
3667                                 rmv_port_callback, (void *)(intptr_t)port_id))
3668                         fprintf(stderr,
3669                                 "Could not set up deferred device removal\n");
3670                 break;
3671         case RTE_ETH_EVENT_DESTROY:
3672                 ports[port_id].port_status = RTE_PORT_CLOSED;
3673                 printf("Port %u is closed\n", port_id);
3674                 break;
3675         default:
3676                 break;
3677         }
3678         return 0;
3679 }
3680
3681 static int
3682 register_eth_event_callback(void)
3683 {
3684         int ret;
3685         enum rte_eth_event_type event;
3686
3687         for (event = RTE_ETH_EVENT_UNKNOWN;
3688                         event < RTE_ETH_EVENT_MAX; event++) {
3689                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3690                                 event,
3691                                 eth_event_callback,
3692                                 NULL);
3693                 if (ret != 0) {
3694                         TESTPMD_LOG(ERR, "Failed to register callback for "
3695                                         "%s event\n", eth_event_desc[event]);
3696                         return -1;
3697                 }
3698         }
3699
3700         return 0;
3701 }
3702
3703 /* This function is used by the interrupt thread */
3704 static void
3705 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3706                              __rte_unused void *arg)
3707 {
3708         uint16_t port_id;
3709         int ret;
3710
3711         if (type >= RTE_DEV_EVENT_MAX) {
3712                 fprintf(stderr, "%s called upon invalid event %d\n",
3713                         __func__, type);
3714                 fflush(stderr);
3715         }
3716
3717         switch (type) {
3718         case RTE_DEV_EVENT_REMOVE:
3719                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3720                         device_name);
3721                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3722                 if (ret) {
3723                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3724                                 device_name);
3725                         return;
3726                 }
3727                 /*
3728                  * Because the user's callback is invoked in eal interrupt
3729                  * callback, the interrupt callback need to be finished before
3730                  * it can be unregistered when detaching device. So finish
3731                  * callback soon and use a deferred removal to detach device
3732                  * is need. It is a workaround, once the device detaching be
3733                  * moved into the eal in the future, the deferred removal could
3734                  * be deleted.
3735                  */
3736                 if (rte_eal_alarm_set(100000,
3737                                 rmv_port_callback, (void *)(intptr_t)port_id))
3738                         RTE_LOG(ERR, EAL,
3739                                 "Could not set up deferred device removal\n");
3740                 break;
3741         case RTE_DEV_EVENT_ADD:
3742                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3743                         device_name);
3744                 /* TODO: After finish kernel driver binding,
3745                  * begin to attach port.
3746                  */
3747                 break;
3748         default:
3749                 break;
3750         }
3751 }
3752
3753 static void
3754 rxtx_port_config(portid_t pid)
3755 {
3756         uint16_t qid;
3757         uint64_t offloads;
3758         struct rte_port *port = &ports[pid];
3759
3760         for (qid = 0; qid < nb_rxq; qid++) {
3761                 offloads = port->rxq[qid].conf.offloads;
3762                 port->rxq[qid].conf = port->dev_info.default_rxconf;
3763
3764                 if (rxq_share > 0 &&
3765                     (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3766                         /* Non-zero share group to enable RxQ share. */
3767                         port->rxq[qid].conf.share_group = pid / rxq_share + 1;
3768                         port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */
3769                 }
3770
3771                 if (offloads != 0)
3772                         port->rxq[qid].conf.offloads = offloads;
3773
3774                 /* Check if any Rx parameters have been passed */
3775                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3776                         port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh;
3777
3778                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3779                         port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh;
3780
3781                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3782                         port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh;
3783
3784                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3785                         port->rxq[qid].conf.rx_free_thresh = rx_free_thresh;
3786
3787                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3788                         port->rxq[qid].conf.rx_drop_en = rx_drop_en;
3789
3790                 port->nb_rx_desc[qid] = nb_rxd;
3791         }
3792
3793         for (qid = 0; qid < nb_txq; qid++) {
3794                 offloads = port->txq[qid].conf.offloads;
3795                 port->txq[qid].conf = port->dev_info.default_txconf;
3796                 if (offloads != 0)
3797                         port->txq[qid].conf.offloads = offloads;
3798
3799                 /* Check if any Tx parameters have been passed */
3800                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3801                         port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh;
3802
3803                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3804                         port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh;
3805
3806                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3807                         port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh;
3808
3809                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3810                         port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh;
3811
3812                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3813                         port->txq[qid].conf.tx_free_thresh = tx_free_thresh;
3814
3815                 port->nb_tx_desc[qid] = nb_txd;
3816         }
3817 }
3818
3819 /*
3820  * Helper function to set MTU from frame size
3821  *
3822  * port->dev_info should be set before calling this function.
3823  *
3824  * return 0 on success, negative on error
3825  */
3826 int
3827 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3828 {
3829         struct rte_port *port = &ports[portid];
3830         uint32_t eth_overhead;
3831         uint16_t mtu, new_mtu;
3832
3833         eth_overhead = get_eth_overhead(&port->dev_info);
3834
3835         if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3836                 printf("Failed to get MTU for port %u\n", portid);
3837                 return -1;
3838         }
3839
3840         new_mtu = max_rx_pktlen - eth_overhead;
3841
3842         if (mtu == new_mtu)
3843                 return 0;
3844
3845         if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3846                 fprintf(stderr,
3847                         "Failed to set MTU to %u for port %u\n",
3848                         new_mtu, portid);
3849                 return -1;
3850         }
3851
3852         port->dev_conf.rxmode.mtu = new_mtu;
3853
3854         return 0;
3855 }
3856
3857 void
3858 init_port_config(void)
3859 {
3860         portid_t pid;
3861         struct rte_port *port;
3862         int ret, i;
3863
3864         RTE_ETH_FOREACH_DEV(pid) {
3865                 port = &ports[pid];
3866                 port->dev_conf.fdir_conf = fdir_conf;
3867
3868                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3869                 if (ret != 0)
3870                         return;
3871
3872                 if (nb_rxq > 1) {
3873                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3874                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3875                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3876                 } else {
3877                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3878                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3879                 }
3880
3881                 if (port->dcb_flag == 0) {
3882                         if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3883                                 port->dev_conf.rxmode.mq_mode =
3884                                         (enum rte_eth_rx_mq_mode)
3885                                                 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3886                         } else {
3887                                 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3888                                 port->dev_conf.rxmode.offloads &=
3889                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3890
3891                                 for (i = 0;
3892                                      i < port->dev_info.nb_rx_queues;
3893                                      i++)
3894                                         port->rxq[i].conf.offloads &=
3895                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3896                         }
3897                 }
3898
3899                 rxtx_port_config(pid);
3900
3901                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3902                 if (ret != 0)
3903                         return;
3904
3905 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3906                 rte_pmd_ixgbe_bypass_init(pid);
3907 #endif
3908
3909                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3910                         port->dev_conf.intr_conf.lsc = 1;
3911                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3912                         port->dev_conf.intr_conf.rmv = 1;
3913         }
3914 }
3915
3916 void set_port_slave_flag(portid_t slave_pid)
3917 {
3918         struct rte_port *port;
3919
3920         port = &ports[slave_pid];
3921         port->slave_flag = 1;
3922 }
3923
3924 void clear_port_slave_flag(portid_t slave_pid)
3925 {
3926         struct rte_port *port;
3927
3928         port = &ports[slave_pid];
3929         port->slave_flag = 0;
3930 }
3931
3932 uint8_t port_is_bonding_slave(portid_t slave_pid)
3933 {
3934         struct rte_port *port;
3935         struct rte_eth_dev_info dev_info;
3936         int ret;
3937
3938         port = &ports[slave_pid];
3939         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3940         if (ret != 0) {
3941                 TESTPMD_LOG(ERR,
3942                         "Failed to get device info for port id %d,"
3943                         "cannot determine if the port is a bonded slave",
3944                         slave_pid);
3945                 return 0;
3946         }
3947         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3948                 return 1;
3949         return 0;
3950 }
3951
3952 const uint16_t vlan_tags[] = {
3953                 0,  1,  2,  3,  4,  5,  6,  7,
3954                 8,  9, 10, 11,  12, 13, 14, 15,
3955                 16, 17, 18, 19, 20, 21, 22, 23,
3956                 24, 25, 26, 27, 28, 29, 30, 31
3957 };
3958
3959 static  int
3960 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3961                  enum dcb_mode_enable dcb_mode,
3962                  enum rte_eth_nb_tcs num_tcs,
3963                  uint8_t pfc_en)
3964 {
3965         uint8_t i;
3966         int32_t rc;
3967         struct rte_eth_rss_conf rss_conf;
3968
3969         /*
3970          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3971          * given above, and the number of traffic classes available for use.
3972          */
3973         if (dcb_mode == DCB_VT_ENABLED) {
3974                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3975                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3976                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3977                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3978
3979                 /* VMDQ+DCB RX and TX configurations */
3980                 vmdq_rx_conf->enable_default_pool = 0;
3981                 vmdq_rx_conf->default_pool = 0;
3982                 vmdq_rx_conf->nb_queue_pools =
3983                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3984                 vmdq_tx_conf->nb_queue_pools =
3985                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3986
3987                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3988                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3989                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3990                         vmdq_rx_conf->pool_map[i].pools =
3991                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3992                 }
3993                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3994                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3995                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3996                 }
3997
3998                 /* set DCB mode of RX and TX of multiple queues */
3999                 eth_conf->rxmode.mq_mode =
4000                                 (enum rte_eth_rx_mq_mode)
4001                                         (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
4002                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
4003         } else {
4004                 struct rte_eth_dcb_rx_conf *rx_conf =
4005                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
4006                 struct rte_eth_dcb_tx_conf *tx_conf =
4007                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
4008
4009                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
4010
4011                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
4012                 if (rc != 0)
4013                         return rc;
4014
4015                 rx_conf->nb_tcs = num_tcs;
4016                 tx_conf->nb_tcs = num_tcs;
4017
4018                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
4019                         rx_conf->dcb_tc[i] = i % num_tcs;
4020                         tx_conf->dcb_tc[i] = i % num_tcs;
4021                 }
4022
4023                 eth_conf->rxmode.mq_mode =
4024                                 (enum rte_eth_rx_mq_mode)
4025                                         (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
4026                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
4027                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
4028         }
4029
4030         if (pfc_en)
4031                 eth_conf->dcb_capability_en =
4032                                 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
4033         else
4034                 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
4035
4036         return 0;
4037 }
4038
4039 int
4040 init_port_dcb_config(portid_t pid,
4041                      enum dcb_mode_enable dcb_mode,
4042                      enum rte_eth_nb_tcs num_tcs,
4043                      uint8_t pfc_en)
4044 {
4045         struct rte_eth_conf port_conf;
4046         struct rte_port *rte_port;
4047         int retval;
4048         uint16_t i;
4049
4050         if (num_procs > 1) {
4051                 printf("The multi-process feature doesn't support dcb.\n");
4052                 return -ENOTSUP;
4053         }
4054         rte_port = &ports[pid];
4055
4056         /* retain the original device configuration. */
4057         memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
4058
4059         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
4060         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
4061         if (retval < 0)
4062                 return retval;
4063         port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4064         /* remove RSS HASH offload for DCB in vt mode */
4065         if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
4066                 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4067                 for (i = 0; i < nb_rxq; i++)
4068                         rte_port->rxq[i].conf.offloads &=
4069                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4070         }
4071
4072         /* re-configure the device . */
4073         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
4074         if (retval < 0)
4075                 return retval;
4076
4077         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
4078         if (retval != 0)
4079                 return retval;
4080
4081         /* If dev_info.vmdq_pool_base is greater than 0,
4082          * the queue id of vmdq pools is started after pf queues.
4083          */
4084         if (dcb_mode == DCB_VT_ENABLED &&
4085             rte_port->dev_info.vmdq_pool_base > 0) {
4086                 fprintf(stderr,
4087                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
4088                         pid);
4089                 return -1;
4090         }
4091
4092         /* Assume the ports in testpmd have the same dcb capability
4093          * and has the same number of rxq and txq in dcb mode
4094          */
4095         if (dcb_mode == DCB_VT_ENABLED) {
4096                 if (rte_port->dev_info.max_vfs > 0) {
4097                         nb_rxq = rte_port->dev_info.nb_rx_queues;
4098                         nb_txq = rte_port->dev_info.nb_tx_queues;
4099                 } else {
4100                         nb_rxq = rte_port->dev_info.max_rx_queues;
4101                         nb_txq = rte_port->dev_info.max_tx_queues;
4102                 }
4103         } else {
4104                 /*if vt is disabled, use all pf queues */
4105                 if (rte_port->dev_info.vmdq_pool_base == 0) {
4106                         nb_rxq = rte_port->dev_info.max_rx_queues;
4107                         nb_txq = rte_port->dev_info.max_tx_queues;
4108                 } else {
4109                         nb_rxq = (queueid_t)num_tcs;
4110                         nb_txq = (queueid_t)num_tcs;
4111
4112                 }
4113         }
4114         rx_free_thresh = 64;
4115
4116         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4117
4118         rxtx_port_config(pid);
4119         /* VLAN filter */
4120         rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4121         for (i = 0; i < RTE_DIM(vlan_tags); i++)
4122                 rx_vft_set(pid, vlan_tags[i], 1);
4123
4124         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4125         if (retval != 0)
4126                 return retval;
4127
4128         rte_port->dcb_flag = 1;
4129
4130         /* Enter DCB configuration status */
4131         dcb_config = 1;
4132
4133         return 0;
4134 }
4135
4136 static void
4137 init_port(void)
4138 {
4139         int i;
4140
4141         /* Configuration of Ethernet ports. */
4142         ports = rte_zmalloc("testpmd: ports",
4143                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4144                             RTE_CACHE_LINE_SIZE);
4145         if (ports == NULL) {
4146                 rte_exit(EXIT_FAILURE,
4147                                 "rte_zmalloc(%d struct rte_port) failed\n",
4148                                 RTE_MAX_ETHPORTS);
4149         }
4150         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4151                 ports[i].xstats_info.allocated = false;
4152         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4153                 LIST_INIT(&ports[i].flow_tunnel_list);
4154         /* Initialize ports NUMA structures */
4155         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4156         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4157         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4158 }
4159
4160 static void
4161 force_quit(void)
4162 {
4163         pmd_test_exit();
4164         prompt_exit();
4165 }
4166
4167 static void
4168 print_stats(void)
4169 {
4170         uint8_t i;
4171         const char clr[] = { 27, '[', '2', 'J', '\0' };
4172         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4173
4174         /* Clear screen and move to top left */
4175         printf("%s%s", clr, top_left);
4176
4177         printf("\nPort statistics ====================================");
4178         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4179                 nic_stats_display(fwd_ports_ids[i]);
4180
4181         fflush(stdout);
4182 }
4183
4184 static void
4185 signal_handler(int signum)
4186 {
4187         if (signum == SIGINT || signum == SIGTERM) {
4188                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4189                         signum);
4190 #ifdef RTE_LIB_PDUMP
4191                 /* uninitialize packet capture framework */
4192                 rte_pdump_uninit();
4193 #endif
4194 #ifdef RTE_LIB_LATENCYSTATS
4195                 if (latencystats_enabled != 0)
4196                         rte_latencystats_uninit();
4197 #endif
4198                 force_quit();
4199                 /* Set flag to indicate the force termination. */
4200                 f_quit = 1;
4201                 /* exit with the expected status */
4202 #ifndef RTE_EXEC_ENV_WINDOWS
4203                 signal(signum, SIG_DFL);
4204                 kill(getpid(), signum);
4205 #endif
4206         }
4207 }
4208
4209 int
4210 main(int argc, char** argv)
4211 {
4212         int diag;
4213         portid_t port_id;
4214         uint16_t count;
4215         int ret;
4216
4217         signal(SIGINT, signal_handler);
4218         signal(SIGTERM, signal_handler);
4219
4220         testpmd_logtype = rte_log_register("testpmd");
4221         if (testpmd_logtype < 0)
4222                 rte_exit(EXIT_FAILURE, "Cannot register log type");
4223         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4224
4225         diag = rte_eal_init(argc, argv);
4226         if (diag < 0)
4227                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4228                          rte_strerror(rte_errno));
4229
4230         ret = register_eth_event_callback();
4231         if (ret != 0)
4232                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4233
4234 #ifdef RTE_LIB_PDUMP
4235         /* initialize packet capture framework */
4236         rte_pdump_init();
4237 #endif
4238
4239         count = 0;
4240         RTE_ETH_FOREACH_DEV(port_id) {
4241                 ports_ids[count] = port_id;
4242                 count++;
4243         }
4244         nb_ports = (portid_t) count;
4245         if (nb_ports == 0)
4246                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4247
4248         /* allocate port structures, and init them */
4249         init_port();
4250
4251         set_def_fwd_config();
4252         if (nb_lcores == 0)
4253                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4254                          "Check the core mask argument\n");
4255
4256         /* Bitrate/latency stats disabled by default */
4257 #ifdef RTE_LIB_BITRATESTATS
4258         bitrate_enabled = 0;
4259 #endif
4260 #ifdef RTE_LIB_LATENCYSTATS
4261         latencystats_enabled = 0;
4262 #endif
4263
4264         /* on FreeBSD, mlockall() is disabled by default */
4265 #ifdef RTE_EXEC_ENV_FREEBSD
4266         do_mlockall = 0;
4267 #else
4268         do_mlockall = 1;
4269 #endif
4270
4271         argc -= diag;
4272         argv += diag;
4273         if (argc > 1)
4274                 launch_args_parse(argc, argv);
4275
4276 #ifndef RTE_EXEC_ENV_WINDOWS
4277         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4278                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4279                         strerror(errno));
4280         }
4281 #endif
4282
4283         if (tx_first && interactive)
4284                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4285                                 "interactive mode.\n");
4286
4287         if (tx_first && lsc_interrupt) {
4288                 fprintf(stderr,
4289                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4290                 lsc_interrupt = 0;
4291         }
4292
4293         if (!nb_rxq && !nb_txq)
4294                 fprintf(stderr,
4295                         "Warning: Either rx or tx queues should be non-zero\n");
4296
4297         if (nb_rxq > 1 && nb_rxq > nb_txq)
4298                 fprintf(stderr,
4299                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4300                         nb_rxq, nb_txq);
4301
4302         init_config();
4303
4304         if (hot_plug) {
4305                 ret = rte_dev_hotplug_handle_enable();
4306                 if (ret) {
4307                         RTE_LOG(ERR, EAL,
4308                                 "fail to enable hotplug handling.");
4309                         return -1;
4310                 }
4311
4312                 ret = rte_dev_event_monitor_start();
4313                 if (ret) {
4314                         RTE_LOG(ERR, EAL,
4315                                 "fail to start device event monitoring.");
4316                         return -1;
4317                 }
4318
4319                 ret = rte_dev_event_callback_register(NULL,
4320                         dev_event_callback, NULL);
4321                 if (ret) {
4322                         RTE_LOG(ERR, EAL,
4323                                 "fail  to register device event callback\n");
4324                         return -1;
4325                 }
4326         }
4327
4328         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4329                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4330
4331         /* set all ports to promiscuous mode by default */
4332         RTE_ETH_FOREACH_DEV(port_id) {
4333                 ret = rte_eth_promiscuous_enable(port_id);
4334                 if (ret != 0)
4335                         fprintf(stderr,
4336                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4337                                 port_id, rte_strerror(-ret));
4338         }
4339
4340 #ifdef RTE_LIB_METRICS
4341         /* Init metrics library */
4342         rte_metrics_init(rte_socket_id());
4343 #endif
4344
4345 #ifdef RTE_LIB_LATENCYSTATS
4346         if (latencystats_enabled != 0) {
4347                 int ret = rte_latencystats_init(1, NULL);
4348                 if (ret)
4349                         fprintf(stderr,
4350                                 "Warning: latencystats init() returned error %d\n",
4351                                 ret);
4352                 fprintf(stderr, "Latencystats running on lcore %d\n",
4353                         latencystats_lcore_id);
4354         }
4355 #endif
4356
4357         /* Setup bitrate stats */
4358 #ifdef RTE_LIB_BITRATESTATS
4359         if (bitrate_enabled != 0) {
4360                 bitrate_data = rte_stats_bitrate_create();
4361                 if (bitrate_data == NULL)
4362                         rte_exit(EXIT_FAILURE,
4363                                 "Could not allocate bitrate data.\n");
4364                 rte_stats_bitrate_reg(bitrate_data);
4365         }
4366 #endif
4367 #ifdef RTE_LIB_CMDLINE
4368         if (init_cmdline() != 0)
4369                 rte_exit(EXIT_FAILURE,
4370                         "Could not initialise cmdline context.\n");
4371
4372         if (strlen(cmdline_filename) != 0)
4373                 cmdline_read_from_file(cmdline_filename);
4374
4375         if (interactive == 1) {
4376                 if (auto_start) {
4377                         printf("Start automatic packet forwarding\n");
4378                         start_packet_forwarding(0);
4379                 }
4380                 prompt();
4381                 pmd_test_exit();
4382         } else
4383 #endif
4384         {
4385                 char c;
4386                 int rc;
4387
4388                 f_quit = 0;
4389
4390                 printf("No commandline core given, start packet forwarding\n");
4391                 start_packet_forwarding(tx_first);
4392                 if (stats_period != 0) {
4393                         uint64_t prev_time = 0, cur_time, diff_time = 0;
4394                         uint64_t timer_period;
4395
4396                         /* Convert to number of cycles */
4397                         timer_period = stats_period * rte_get_timer_hz();
4398
4399                         while (f_quit == 0) {
4400                                 cur_time = rte_get_timer_cycles();
4401                                 diff_time += cur_time - prev_time;
4402
4403                                 if (diff_time >= timer_period) {
4404                                         print_stats();
4405                                         /* Reset the timer */
4406                                         diff_time = 0;
4407                                 }
4408                                 /* Sleep to avoid unnecessary checks */
4409                                 prev_time = cur_time;
4410                                 rte_delay_us_sleep(US_PER_S);
4411                         }
4412                 }
4413
4414                 printf("Press enter to exit\n");
4415                 rc = read(0, &c, 1);
4416                 pmd_test_exit();
4417                 if (rc < 0)
4418                         return 1;
4419         }
4420
4421         ret = rte_eal_cleanup();
4422         if (ret != 0)
4423                 rte_exit(EXIT_FAILURE,
4424                          "EAL cleanup failed: %s\n", strerror(-ret));
4425
4426         return EXIT_SUCCESS;
4427 }