app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_NET_IXGBE
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIB_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #ifdef RTE_LIB_METRICS
  58 #include <rte_metrics.h>
  59 #endif
  60 #ifdef RTE_LIB_BITRATESTATS
  61 #include <rte_bitrate.h>
  62 #endif
  63 #ifdef RTE_LIB_LATENCYSTATS
  64 #include <rte_latencystats.h>
  65 #endif
  66 #ifdef RTE_EXEC_ENV_WINDOWS
  67 #include <process.h>
  68 #endif
  69 #ifdef RTE_NET_BOND
  70 #include <rte_eth_bond.h>
  71 #endif
  72
  73 #include "testpmd.h"
  74
  75 #ifndef MAP_HUGETLB
  76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  77 #define HUGE_FLAG (0x40000)
  78 #else
  79 #define HUGE_FLAG MAP_HUGETLB
  80 #endif
  81
  82 #ifndef MAP_HUGE_SHIFT
  83 /* older kernels (or FreeBSD) will not have this define */
  84 #define HUGE_SHIFT (26)
  85 #else
  86 #define HUGE_SHIFT MAP_HUGE_SHIFT
  87 #endif
  88
  89 #define EXTMEM_HEAP_NAME "extmem"
  90 /*
  91  * Zone size with the malloc overhead (max of debug and release variants)
  92  * must fit into the smallest supported hugepage size (2M),
  93  * so that an IOVA-contiguous zone of this size can always be allocated
  94  * if there are free 2M hugepages.
  95  */
  96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE)
  97
  98 uint16_t verbose_level = 0; /**< Silent by default. */
  99 int testpmd_logtype; /**< Log type for testpmd logs */
 100
 101 /* use main core for command line ? */
 102 uint8_t interactive = 0;
 103 uint8_t auto_start = 0;
 104 uint8_t tx_first;
 105 char cmdline_filename[PATH_MAX] = {0};
 106
 107 /*
 108  * NUMA support configuration.
 109  * When set, the NUMA support attempts to dispatch the allocation of the
 110  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 111  * probed ports among the CPU sockets 0 and 1.
 112  * Otherwise, all memory is allocated from CPU socket 0.
 113  */
 114 uint8_t numa_support = 1; /**< numa enabled by default */
 115
 116 /*
 117  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 118  * not configured.
 119  */
 120 uint8_t socket_num = UMA_NO_CONFIG;
 121
 122 /*
 123  * Select mempool allocation type:
 124  * - native: use regular DPDK memory
 125  * - anon: use regular DPDK memory to create mempool, but populate using
 126  *         anonymous memory (may not be IOVA-contiguous)
 127  * - xmem: use externally allocated hugepage memory
 128  */
 129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 130
 131 /*
 132  * Store specified sockets on which memory pool to be used by ports
 133  * is allocated.
 134  */
 135 uint8_t port_numa[RTE_MAX_ETHPORTS];
 136
 137 /*
 138  * Store specified sockets on which RX ring to be used by ports
 139  * is allocated.
 140  */
 141 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 142
 143 /*
 144  * Store specified sockets on which TX ring to be used by ports
 145  * is allocated.
 146  */
 147 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 148
 149 /*
 150  * Record the Ethernet address of peer target ports to which packets are
 151  * forwarded.
 152  * Must be instantiated with the ethernet addresses of peer traffic generator
 153  * ports.
 154  */
 155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 156 portid_t nb_peer_eth_addrs = 0;
 157
 158 /*
 159  * Probed Target Environment.
 160  */
 161 struct rte_port *ports;        /**< For all probed ethernet ports. */
 162 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 164 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 165
 166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 167
 168 /*
 169  * Test Forwarding Configuration.
 170  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 171  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 172  */
 173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 175 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 176 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 177
 178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 180
 181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 182 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 183
 184 /*
 185  * Forwarding engines.
 186  */
 187 struct fwd_engine * fwd_engines[] = {
 188         &io_fwd_engine,
 189         &mac_fwd_engine,
 190         &mac_swap_engine,
 191         &flow_gen_engine,
 192         &rx_only_engine,
 193         &tx_only_engine,
 194         &csum_fwd_engine,
 195         &icmp_echo_engine,
 196         &noisy_vnf_engine,
 197         &five_tuple_swap_fwd_engine,
 198 #ifdef RTE_LIBRTE_IEEE1588
 199         &ieee1588_fwd_engine,
 200 #endif
 201         &shared_rxq_engine,
 202         NULL,
 203 };
 204
 205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 206 uint16_t mempool_flags;
 207
 208 struct fwd_config cur_fwd_config;
 209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 210 uint32_t retry_enabled;
 211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 213
 214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 216         DEFAULT_MBUF_DATA_SIZE
 217 }; /**< Mbuf data space size. */
 218 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 219                                       * specified on command-line. */
 220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 221
 222 /** Extended statistics to show. */
 223 struct rte_eth_xstat_name *xstats_display;
 224
 225 unsigned int xstats_display_num; /**< Size of extended statistics to show */
 226
 227 /*
 228  * In container, it cannot terminate the process which running with 'stats-period'
 229  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 230  */
 231 uint8_t f_quit;
 232
 233 /*
 234  * Max Rx frame size, set by '--max-pkt-len' parameter.
 235  */
 236 uint32_t max_rx_pkt_len;
 237
 238 /*
 239  * Configuration of packet segments used to scatter received packets
 240  * if some of split features is configured.
 241  */
 242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 243 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 245 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 246
 247 /*
 248  * Configuration of packet segments used by the "txonly" processing engine.
 249  */
 250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 252         TXONLY_DEF_PACKET_LEN,
 253 };
 254 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 255
 256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 257 /**< Split policy for packets to TX. */
 258
 259 uint8_t txonly_multi_flow;
 260 /**< Whether multiple flows are generated in TXONLY mode. */
 261
 262 uint32_t tx_pkt_times_inter;
 263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 264
 265 uint32_t tx_pkt_times_intra;
 266 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 267
 268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
 271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 272
 273 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 274 uint8_t dcb_config = 0;
 275
 276 /*
 277  * Configurable number of RX/TX queues.
 278  */
 279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 282
 283 /*
 284  * Configurable number of RX/TX ring descriptors.
 285  * Defaults are supplied by drivers via ethdev.
 286  */
 287 #define RTE_TEST_RX_DESC_DEFAULT 0
 288 #define RTE_TEST_TX_DESC_DEFAULT 0
 289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 291
 292 #define RTE_PMD_PARAM_UNSET -1
 293 /*
 294  * Configurable values of RX and TX ring threshold registers.
 295  */
 296
 297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 300
 301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 304
 305 /*
 306  * Configurable value of RX free threshold.
 307  */
 308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 309
 310 /*
 311  * Configurable value of RX drop enable.
 312  */
 313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 314
 315 /*
 316  * Configurable value of TX free threshold.
 317  */
 318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 319
 320 /*
 321  * Configurable value of TX RS bit threshold.
 322  */
 323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 324
 325 /*
 326  * Configurable value of buffered packets before sending.
 327  */
 328 uint16_t noisy_tx_sw_bufsz;
 329
 330 /*
 331  * Configurable value of packet buffer timeout.
 332  */
 333 uint16_t noisy_tx_sw_buf_flush_time;
 334
 335 /*
 336  * Configurable value for size of VNF internal memory area
 337  * used for simulating noisy neighbour behaviour
 338  */
 339 uint64_t noisy_lkup_mem_sz;
 340
 341 /*
 342  * Configurable value of number of random writes done in
 343  * VNF simulation memory area.
 344  */
 345 uint64_t noisy_lkup_num_writes;
 346
 347 /*
 348  * Configurable value of number of random reads done in
 349  * VNF simulation memory area.
 350  */
 351 uint64_t noisy_lkup_num_reads;
 352
 353 /*
 354  * Configurable value of number of random reads/writes done in
 355  * VNF simulation memory area.
 356  */
 357 uint64_t noisy_lkup_num_reads_writes;
 358
 359 /*
 360  * Receive Side Scaling (RSS) configuration.
 361  */
 362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */
 363
 364 /*
 365  * Port topology configuration
 366  */
 367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 368
 369 /*
 370  * Avoids to flush all the RX streams before starts forwarding.
 371  */
 372 uint8_t no_flush_rx = 0; /* flush by default */
 373
 374 /*
 375  * Flow API isolated mode.
 376  */
 377 uint8_t flow_isolate_all;
 378
 379 /*
 380  * Avoids to check link status when starting/stopping a port.
 381  */
 382 uint8_t no_link_check = 0; /* check by default */
 383
 384 /*
 385  * Don't automatically start all ports in interactive mode.
 386  */
 387 uint8_t no_device_start = 0;
 388
 389 /*
 390  * Enable link status change notification
 391  */
 392 uint8_t lsc_interrupt = 1; /* enabled by default */
 393
 394 /*
 395  * Enable device removal notification.
 396  */
 397 uint8_t rmv_interrupt = 1; /* enabled by default */
 398
 399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 400
 401 /* After attach, port setup is called on event or by iterator */
 402 bool setup_on_probe_event = true;
 403
 404 /* Clear ptypes on port initialization. */
 405 uint8_t clear_ptypes = true;
 406
 407 /* Hairpin ports configuration mode. */
 408 uint16_t hairpin_mode;
 409
 410 /* Pretty printing of ethdev events */
 411 static const char * const eth_event_desc[] = {
 412         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 413         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 414         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 415         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 416         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 417         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 418         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 419         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 420         [RTE_ETH_EVENT_NEW] = "device probed",
 421         [RTE_ETH_EVENT_DESTROY] = "device released",
 422         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 423         [RTE_ETH_EVENT_MAX] = NULL,
 424 };
 425
 426 /*
 427  * Display or mask ether events
 428  * Default to all events except VF_MBOX
 429  */
 430 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 431                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 432                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 433                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 434                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 435                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 436                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 437                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 438 /*
 439  * Decide if all memory are locked for performance.
 440  */
 441 int do_mlockall = 0;
 442
 443 /*
 444  * NIC bypass mode configuration options.
 445  */
 446
 447 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 448 /* The NIC bypass watchdog timeout. */
 449 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 450 #endif
 451
 452
 453 #ifdef RTE_LIB_LATENCYSTATS
 454
 455 /*
 456  * Set when latency stats is enabled in the commandline
 457  */
 458 uint8_t latencystats_enabled;
 459
 460 /*
 461  * Lcore ID to service latency statistics.
 462  */
 463 lcoreid_t latencystats_lcore_id = -1;
 464
 465 #endif
 466
 467 /*
 468  * Ethernet device configuration.
 469  */
 470 struct rte_eth_rxmode rx_mode;
 471
 472 struct rte_eth_txmode tx_mode = {
 473         .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
 474 };
 475
 476 struct rte_eth_fdir_conf fdir_conf = {
 477         .mode = RTE_FDIR_MODE_NONE,
 478         .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
 479         .status = RTE_FDIR_REPORT_STATUS,
 480         .mask = {
 481                 .vlan_tci_mask = 0xFFEF,
 482                 .ipv4_mask     = {
 483                         .src_ip = 0xFFFFFFFF,
 484                         .dst_ip = 0xFFFFFFFF,
 485                 },
 486                 .ipv6_mask     = {
 487                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 488                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 489                 },
 490                 .src_port_mask = 0xFFFF,
 491                 .dst_port_mask = 0xFFFF,
 492                 .mac_addr_byte_mask = 0xFF,
 493                 .tunnel_type_mask = 1,
 494                 .tunnel_id_mask = 0xFFFFFFFF,
 495         },
 496         .drop_queue = 127,
 497 };
 498
 499 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 500
 501 /*
 502  * Display zero values by default for xstats
 503  */
 504 uint8_t xstats_hide_zero;
 505
 506 /*
 507  * Measure of CPU cycles disabled by default
 508  */
 509 uint8_t record_core_cycles;
 510
 511 /*
 512  * Display of RX and TX bursts disabled by default
 513  */
 514 uint8_t record_burst_stats;
 515
 516 /*
 517  * Number of ports per shared Rx queue group, 0 disable.
 518  */
 519 uint32_t rxq_share;
 520
 521 unsigned int num_sockets = 0;
 522 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 523
 524 #ifdef RTE_LIB_BITRATESTATS
 525 /* Bitrate statistics */
 526 struct rte_stats_bitrates *bitrate_data;
 527 lcoreid_t bitrate_lcore_id;
 528 uint8_t bitrate_enabled;
 529 #endif
 530
 531 #ifdef RTE_LIB_GRO
 532 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 533 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 534 #endif
 535
 536 /*
 537  * hexadecimal bitmask of RX mq mode can be enabled.
 538  */
 539 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
 540
 541 /*
 542  * Used to set forced link speed
 543  */
 544 uint32_t eth_link_speed;
 545
 546 /*
 547  * ID of the current process in multi-process, used to
 548  * configure the queues to be polled.
 549  */
 550 int proc_id;
 551
 552 /*
 553  * Number of processes in multi-process, used to
 554  * configure the queues to be polled.
 555  */
 556 unsigned int num_procs = 1;
 557
 558 static void
 559 eth_rx_metadata_negotiate_mp(uint16_t port_id)
 560 {
 561         uint64_t rx_meta_features = 0;
 562         int ret;
 563
 564         if (!is_proc_primary())
 565                 return;
 566
 567         rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
 568         rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
 569         rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
 570
 571         ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
 572         if (ret == 0) {
 573                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
 574                         TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
 575                                     port_id);
 576                 }
 577
 578                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
 579                         TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
 580                                     port_id);
 581                 }
 582
 583                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
 584                         TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
 585                                     port_id);
 586                 }
 587         } else if (ret != -ENOTSUP) {
 588                 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
 589                          port_id, rte_strerror(-ret));
 590         }
 591 }
 592
 593 static int
 594 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 595                       const struct rte_eth_conf *dev_conf)
 596 {
 597         if (is_proc_primary())
 598                 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
 599                                         dev_conf);
 600         return 0;
 601 }
 602
 603 static int
 604 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop)
 605 {
 606 #ifdef RTE_NET_BOND
 607
 608         portid_t slave_pids[RTE_MAX_ETHPORTS];
 609         struct rte_port *port;
 610         int num_slaves;
 611         portid_t slave_pid;
 612         int i;
 613
 614         num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids,
 615                                                 RTE_MAX_ETHPORTS);
 616         if (num_slaves < 0) {
 617                 fprintf(stderr, "Failed to get slave list for port = %u\n",
 618                         bond_pid);
 619                 return num_slaves;
 620         }
 621
 622         for (i = 0; i < num_slaves; i++) {
 623                 slave_pid = slave_pids[i];
 624                 port = &ports[slave_pid];
 625                 port->port_status =
 626                         is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED;
 627         }
 628 #else
 629         RTE_SET_USED(bond_pid);
 630         RTE_SET_USED(is_stop);
 631 #endif
 632         return 0;
 633 }
 634
 635 static int
 636 eth_dev_start_mp(uint16_t port_id)
 637 {
 638         int ret;
 639
 640         if (is_proc_primary()) {
 641                 ret = rte_eth_dev_start(port_id);
 642                 if (ret != 0)
 643                         return ret;
 644
 645                 struct rte_port *port = &ports[port_id];
 646
 647                 /*
 648                  * Starting a bonded port also starts all slaves under the bonded
 649                  * device. So if this port is bond device, we need to modify the
 650                  * port status of these slaves.
 651                  */
 652                 if (port->bond_flag == 1)
 653                         return change_bonding_slave_port_status(port_id, false);
 654         }
 655
 656         return 0;
 657 }
 658
 659 static int
 660 eth_dev_stop_mp(uint16_t port_id)
 661 {
 662         int ret;
 663
 664         if (is_proc_primary()) {
 665                 ret = rte_eth_dev_stop(port_id);
 666                 if (ret != 0)
 667                         return ret;
 668
 669                 struct rte_port *port = &ports[port_id];
 670
 671                 /*
 672                  * Stopping a bonded port also stops all slaves under the bonded
 673                  * device. So if this port is bond device, we need to modify the
 674                  * port status of these slaves.
 675                  */
 676                 if (port->bond_flag == 1)
 677                         return change_bonding_slave_port_status(port_id, true);
 678         }
 679
 680         return 0;
 681 }
 682
 683 static void
 684 mempool_free_mp(struct rte_mempool *mp)
 685 {
 686         if (is_proc_primary())
 687                 rte_mempool_free(mp);
 688 }
 689
 690 static int
 691 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
 692 {
 693         if (is_proc_primary())
 694                 return rte_eth_dev_set_mtu(port_id, mtu);
 695
 696         return 0;
 697 }
 698
 699 /* Forward function declarations */
 700 static void setup_attached_port(portid_t pi);
 701 static void check_all_ports_link_status(uint32_t port_mask);
 702 static int eth_event_callback(portid_t port_id,
 703                               enum rte_eth_event_type type,
 704                               void *param, void *ret_param);
 705 static void dev_event_callback(const char *device_name,
 706                                 enum rte_dev_event_type type,
 707                                 void *param);
 708 static void fill_xstats_display_info(void);
 709
 710 /*
 711  * Check if all the ports are started.
 712  * If yes, return positive value. If not, return zero.
 713  */
 714 static int all_ports_started(void);
 715
 716 #ifdef RTE_LIB_GSO
 717 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 718 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 719 #endif
 720
 721 /* Holds the registered mbuf dynamic flags names. */
 722 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 723
 724
 725 /*
 726  * Helper function to check if socket is already discovered.
 727  * If yes, return positive value. If not, return zero.
 728  */
 729 int
 730 new_socket_id(unsigned int socket_id)
 731 {
 732         unsigned int i;
 733
 734         for (i = 0; i < num_sockets; i++) {
 735                 if (socket_ids[i] == socket_id)
 736                         return 0;
 737         }
 738         return 1;
 739 }
 740
 741 /*
 742  * Setup default configuration.
 743  */
 744 static void
 745 set_default_fwd_lcores_config(void)
 746 {
 747         unsigned int i;
 748         unsigned int nb_lc;
 749         unsigned int sock_num;
 750
 751         nb_lc = 0;
 752         for (i = 0; i < RTE_MAX_LCORE; i++) {
 753                 if (!rte_lcore_is_enabled(i))
 754                         continue;
 755                 sock_num = rte_lcore_to_socket_id(i);
 756                 if (new_socket_id(sock_num)) {
 757                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 758                                 rte_exit(EXIT_FAILURE,
 759                                          "Total sockets greater than %u\n",
 760                                          RTE_MAX_NUMA_NODES);
 761                         }
 762                         socket_ids[num_sockets++] = sock_num;
 763                 }
 764                 if (i == rte_get_main_lcore())
 765                         continue;
 766                 fwd_lcores_cpuids[nb_lc++] = i;
 767         }
 768         nb_lcores = (lcoreid_t) nb_lc;
 769         nb_cfg_lcores = nb_lcores;
 770         nb_fwd_lcores = 1;
 771 }
 772
 773 static void
 774 set_def_peer_eth_addrs(void)
 775 {
 776         portid_t i;
 777
 778         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 779                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 780                 peer_eth_addrs[i].addr_bytes[5] = i;
 781         }
 782 }
 783
 784 static void
 785 set_default_fwd_ports_config(void)
 786 {
 787         portid_t pt_id;
 788         int i = 0;
 789
 790         RTE_ETH_FOREACH_DEV(pt_id) {
 791                 fwd_ports_ids[i++] = pt_id;
 792
 793                 /* Update sockets info according to the attached device */
 794                 int socket_id = rte_eth_dev_socket_id(pt_id);
 795                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 796                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 797                                 rte_exit(EXIT_FAILURE,
 798                                          "Total sockets greater than %u\n",
 799                                          RTE_MAX_NUMA_NODES);
 800                         }
 801                         socket_ids[num_sockets++] = socket_id;
 802                 }
 803         }
 804
 805         nb_cfg_ports = nb_ports;
 806         nb_fwd_ports = nb_ports;
 807 }
 808
 809 void
 810 set_def_fwd_config(void)
 811 {
 812         set_default_fwd_lcores_config();
 813         set_def_peer_eth_addrs();
 814         set_default_fwd_ports_config();
 815 }
 816
 817 #ifndef RTE_EXEC_ENV_WINDOWS
 818 /* extremely pessimistic estimation of memory required to create a mempool */
 819 static int
 820 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 821 {
 822         unsigned int n_pages, mbuf_per_pg, leftover;
 823         uint64_t total_mem, mbuf_mem, obj_sz;
 824
 825         /* there is no good way to predict how much space the mempool will
 826          * occupy because it will allocate chunks on the fly, and some of those
 827          * will come from default DPDK memory while some will come from our
 828          * external memory, so just assume 128MB will be enough for everyone.
 829          */
 830         uint64_t hdr_mem = 128 << 20;
 831
 832         /* account for possible non-contiguousness */
 833         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 834         if (obj_sz > pgsz) {
 835                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 836                 return -1;
 837         }
 838
 839         mbuf_per_pg = pgsz / obj_sz;
 840         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 841         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 842
 843         mbuf_mem = n_pages * pgsz;
 844
 845         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 846
 847         if (total_mem > SIZE_MAX) {
 848                 TESTPMD_LOG(ERR, "Memory size too big\n");
 849                 return -1;
 850         }
 851         *out = (size_t)total_mem;
 852
 853         return 0;
 854 }
 855
 856 static int
 857 pagesz_flags(uint64_t page_sz)
 858 {
 859         /* as per mmap() manpage, all page sizes are log2 of page size
 860          * shifted by MAP_HUGE_SHIFT
 861          */
 862         int log2 = rte_log2_u64(page_sz);
 863
 864         return (log2 << HUGE_SHIFT);
 865 }
 866
 867 static void *
 868 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 869 {
 870         void *addr;
 871         int flags;
 872
 873         /* allocate anonymous hugepages */
 874         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 875         if (huge)
 876                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 877
 878         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 879         if (addr == MAP_FAILED)
 880                 return NULL;
 881
 882         return addr;
 883 }
 884
 885 struct extmem_param {
 886         void *addr;
 887         size_t len;
 888         size_t pgsz;
 889         rte_iova_t *iova_table;
 890         unsigned int iova_table_len;
 891 };
 892
 893 static int
 894 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 895                 bool huge)
 896 {
 897         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 898                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 899         unsigned int cur_page, n_pages, pgsz_idx;
 900         size_t mem_sz, cur_pgsz;
 901         rte_iova_t *iovas = NULL;
 902         void *addr;
 903         int ret;
 904
 905         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 906                 /* skip anything that is too big */
 907                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 908                         continue;
 909
 910                 cur_pgsz = pgsizes[pgsz_idx];
 911
 912                 /* if we were told not to allocate hugepages, override */
 913                 if (!huge)
 914                         cur_pgsz = sysconf(_SC_PAGESIZE);
 915
 916                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 917                 if (ret < 0) {
 918                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 919                         return -1;
 920                 }
 921
 922                 /* allocate our memory */
 923                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 924
 925                 /* if we couldn't allocate memory with a specified page size,
 926                  * that doesn't mean we can't do it with other page sizes, so
 927                  * try another one.
 928                  */
 929                 if (addr == NULL)
 930                         continue;
 931
 932                 /* store IOVA addresses for every page in this memory area */
 933                 n_pages = mem_sz / cur_pgsz;
 934
 935                 iovas = malloc(sizeof(*iovas) * n_pages);
 936
 937                 if (iovas == NULL) {
 938                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 939                         goto fail;
 940                 }
 941                 /* lock memory if it's not huge pages */
 942                 if (!huge)
 943                         mlock(addr, mem_sz);
 944
 945                 /* populate IOVA addresses */
 946                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 947                         rte_iova_t iova;
 948                         size_t offset;
 949                         void *cur;
 950
 951                         offset = cur_pgsz * cur_page;
 952                         cur = RTE_PTR_ADD(addr, offset);
 953
 954                         /* touch the page before getting its IOVA */
 955                         *(volatile char *)cur = 0;
 956
 957                         iova = rte_mem_virt2iova(cur);
 958
 959                         iovas[cur_page] = iova;
 960                 }
 961
 962                 break;
 963         }
 964         /* if we couldn't allocate anything */
 965         if (iovas == NULL)
 966                 return -1;
 967
 968         param->addr = addr;
 969         param->len = mem_sz;
 970         param->pgsz = cur_pgsz;
 971         param->iova_table = iovas;
 972         param->iova_table_len = n_pages;
 973
 974         return 0;
 975 fail:
 976         free(iovas);
 977         if (addr)
 978                 munmap(addr, mem_sz);
 979
 980         return -1;
 981 }
 982
 983 static int
 984 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 985 {
 986         struct extmem_param param;
 987         int socket_id, ret;
 988
 989         memset(&param, 0, sizeof(param));
 990
 991         /* check if our heap exists */
 992         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 993         if (socket_id < 0) {
 994                 /* create our heap */
 995                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 996                 if (ret < 0) {
 997                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 998                         return -1;
 999                 }
1000         }
1001
1002         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
1003         if (ret < 0) {
1004                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
1005                 return -1;
1006         }
1007
1008         /* we now have a valid memory area, so add it to heap */
1009         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
1010                         param.addr, param.len, param.iova_table,
1011                         param.iova_table_len, param.pgsz);
1012
1013         /* when using VFIO, memory is automatically mapped for DMA by EAL */
1014
1015         /* not needed any more */
1016         free(param.iova_table);
1017
1018         if (ret < 0) {
1019                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
1020                 munmap(param.addr, param.len);
1021                 return -1;
1022         }
1023
1024         /* success */
1025
1026         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
1027                         param.len >> 20);
1028
1029         return 0;
1030 }
1031 static void
1032 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1033              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1034 {
1035         uint16_t pid = 0;
1036         int ret;
1037
1038         RTE_ETH_FOREACH_DEV(pid) {
1039                 struct rte_eth_dev_info dev_info;
1040
1041                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1042                 if (ret != 0) {
1043                         TESTPMD_LOG(DEBUG,
1044                                     "unable to get device info for port %d on addr 0x%p,"
1045                                     "mempool unmapping will not be performed\n",
1046                                     pid, memhdr->addr);
1047                         continue;
1048                 }
1049
1050                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
1051                 if (ret) {
1052                         TESTPMD_LOG(DEBUG,
1053                                     "unable to DMA unmap addr 0x%p "
1054                                     "for device %s\n",
1055                                     memhdr->addr, dev_info.device->name);
1056                 }
1057         }
1058         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
1059         if (ret) {
1060                 TESTPMD_LOG(DEBUG,
1061                             "unable to un-register addr 0x%p\n", memhdr->addr);
1062         }
1063 }
1064
1065 static void
1066 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1067            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1068 {
1069         uint16_t pid = 0;
1070         size_t page_size = sysconf(_SC_PAGESIZE);
1071         int ret;
1072
1073         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1074                                   page_size);
1075         if (ret) {
1076                 TESTPMD_LOG(DEBUG,
1077                             "unable to register addr 0x%p\n", memhdr->addr);
1078                 return;
1079         }
1080         RTE_ETH_FOREACH_DEV(pid) {
1081                 struct rte_eth_dev_info dev_info;
1082
1083                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1084                 if (ret != 0) {
1085                         TESTPMD_LOG(DEBUG,
1086                                     "unable to get device info for port %d on addr 0x%p,"
1087                                     "mempool mapping will not be performed\n",
1088                                     pid, memhdr->addr);
1089                         continue;
1090                 }
1091                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1092                 if (ret) {
1093                         TESTPMD_LOG(DEBUG,
1094                                     "unable to DMA map addr 0x%p "
1095                                     "for device %s\n",
1096                                     memhdr->addr, dev_info.device->name);
1097                 }
1098         }
1099 }
1100 #endif
1101
1102 static unsigned int
1103 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1104             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1105 {
1106         struct rte_pktmbuf_extmem *xmem;
1107         unsigned int ext_num, zone_num, elt_num;
1108         uint16_t elt_size;
1109
1110         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1111         elt_num = EXTBUF_ZONE_SIZE / elt_size;
1112         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1113
1114         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1115         if (xmem == NULL) {
1116                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1117                                  "external buffer descriptors\n");
1118                 *ext_mem = NULL;
1119                 return 0;
1120         }
1121         for (ext_num = 0; ext_num < zone_num; ext_num++) {
1122                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1123                 const struct rte_memzone *mz;
1124                 char mz_name[RTE_MEMZONE_NAMESIZE];
1125                 int ret;
1126
1127                 ret = snprintf(mz_name, sizeof(mz_name),
1128                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1129                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1130                         errno = ENAMETOOLONG;
1131                         ext_num = 0;
1132                         break;
1133                 }
1134                 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE,
1135                                          socket_id,
1136                                          RTE_MEMZONE_IOVA_CONTIG |
1137                                          RTE_MEMZONE_1GB |
1138                                          RTE_MEMZONE_SIZE_HINT_ONLY);
1139                 if (mz == NULL) {
1140                         /*
1141                          * The caller exits on external buffer creation
1142                          * error, so there is no need to free memzones.
1143                          */
1144                         errno = ENOMEM;
1145                         ext_num = 0;
1146                         break;
1147                 }
1148                 xseg->buf_ptr = mz->addr;
1149                 xseg->buf_iova = mz->iova;
1150                 xseg->buf_len = EXTBUF_ZONE_SIZE;
1151                 xseg->elt_size = elt_size;
1152         }
1153         if (ext_num == 0 && xmem != NULL) {
1154                 free(xmem);
1155                 xmem = NULL;
1156         }
1157         *ext_mem = xmem;
1158         return ext_num;
1159 }
1160
1161 /*
1162  * Configuration initialisation done once at init time.
1163  */
1164 static struct rte_mempool *
1165 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1166                  unsigned int socket_id, uint16_t size_idx)
1167 {
1168         char pool_name[RTE_MEMPOOL_NAMESIZE];
1169         struct rte_mempool *rte_mp = NULL;
1170 #ifndef RTE_EXEC_ENV_WINDOWS
1171         uint32_t mb_size;
1172
1173         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1174 #endif
1175         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1176         if (!is_proc_primary()) {
1177                 rte_mp = rte_mempool_lookup(pool_name);
1178                 if (rte_mp == NULL)
1179                         rte_exit(EXIT_FAILURE,
1180                                 "Get mbuf pool for socket %u failed: %s\n",
1181                                 socket_id, rte_strerror(rte_errno));
1182                 return rte_mp;
1183         }
1184
1185         TESTPMD_LOG(INFO,
1186                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1187                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1188
1189         switch (mp_alloc_type) {
1190         case MP_ALLOC_NATIVE:
1191                 {
1192                         /* wrapper to rte_mempool_create() */
1193                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1194                                         rte_mbuf_best_mempool_ops());
1195                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1196                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1197                         break;
1198                 }
1199 #ifndef RTE_EXEC_ENV_WINDOWS
1200         case MP_ALLOC_ANON:
1201                 {
1202                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1203                                 mb_size, (unsigned int) mb_mempool_cache,
1204                                 sizeof(struct rte_pktmbuf_pool_private),
1205                                 socket_id, mempool_flags);
1206                         if (rte_mp == NULL)
1207                                 goto err;
1208
1209                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1210                                 rte_mempool_free(rte_mp);
1211                                 rte_mp = NULL;
1212                                 goto err;
1213                         }
1214                         rte_pktmbuf_pool_init(rte_mp, NULL);
1215                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1216                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1217                         break;
1218                 }
1219         case MP_ALLOC_XMEM:
1220         case MP_ALLOC_XMEM_HUGE:
1221                 {
1222                         int heap_socket;
1223                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1224
1225                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1226                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1227
1228                         heap_socket =
1229                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1230                         if (heap_socket < 0)
1231                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1232
1233                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1234                                         rte_mbuf_best_mempool_ops());
1235                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1236                                         mb_mempool_cache, 0, mbuf_seg_size,
1237                                         heap_socket);
1238                         break;
1239                 }
1240 #endif
1241         case MP_ALLOC_XBUF:
1242                 {
1243                         struct rte_pktmbuf_extmem *ext_mem;
1244                         unsigned int ext_num;
1245
1246                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1247                                                socket_id, pool_name, &ext_mem);
1248                         if (ext_num == 0)
1249                                 rte_exit(EXIT_FAILURE,
1250                                          "Can't create pinned data buffers\n");
1251
1252                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1253                                         rte_mbuf_best_mempool_ops());
1254                         rte_mp = rte_pktmbuf_pool_create_extbuf
1255                                         (pool_name, nb_mbuf, mb_mempool_cache,
1256                                          0, mbuf_seg_size, socket_id,
1257                                          ext_mem, ext_num);
1258                         free(ext_mem);
1259                         break;
1260                 }
1261         default:
1262                 {
1263                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1264                 }
1265         }
1266
1267 #ifndef RTE_EXEC_ENV_WINDOWS
1268 err:
1269 #endif
1270         if (rte_mp == NULL) {
1271                 rte_exit(EXIT_FAILURE,
1272                         "Creation of mbuf pool for socket %u failed: %s\n",
1273                         socket_id, rte_strerror(rte_errno));
1274         } else if (verbose_level > 0) {
1275                 rte_mempool_dump(stdout, rte_mp);
1276         }
1277         return rte_mp;
1278 }
1279
1280 /*
1281  * Check given socket id is valid or not with NUMA mode,
1282  * if valid, return 0, else return -1
1283  */
1284 static int
1285 check_socket_id(const unsigned int socket_id)
1286 {
1287         static int warning_once = 0;
1288
1289         if (new_socket_id(socket_id)) {
1290                 if (!warning_once && numa_support)
1291                         fprintf(stderr,
1292                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1293                 warning_once = 1;
1294                 return -1;
1295         }
1296         return 0;
1297 }
1298
1299 /*
1300  * Get the allowed maximum number of RX queues.
1301  * *pid return the port id which has minimal value of
1302  * max_rx_queues in all ports.
1303  */
1304 queueid_t
1305 get_allowed_max_nb_rxq(portid_t *pid)
1306 {
1307         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1308         bool max_rxq_valid = false;
1309         portid_t pi;
1310         struct rte_eth_dev_info dev_info;
1311
1312         RTE_ETH_FOREACH_DEV(pi) {
1313                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1314                         continue;
1315
1316                 max_rxq_valid = true;
1317                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1318                         allowed_max_rxq = dev_info.max_rx_queues;
1319                         *pid = pi;
1320                 }
1321         }
1322         return max_rxq_valid ? allowed_max_rxq : 0;
1323 }
1324
1325 /*
1326  * Check input rxq is valid or not.
1327  * If input rxq is not greater than any of maximum number
1328  * of RX queues of all ports, it is valid.
1329  * if valid, return 0, else return -1
1330  */
1331 int
1332 check_nb_rxq(queueid_t rxq)
1333 {
1334         queueid_t allowed_max_rxq;
1335         portid_t pid = 0;
1336
1337         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1338         if (rxq > allowed_max_rxq) {
1339                 fprintf(stderr,
1340                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1341                         rxq, allowed_max_rxq, pid);
1342                 return -1;
1343         }
1344         return 0;
1345 }
1346
1347 /*
1348  * Get the allowed maximum number of TX queues.
1349  * *pid return the port id which has minimal value of
1350  * max_tx_queues in all ports.
1351  */
1352 queueid_t
1353 get_allowed_max_nb_txq(portid_t *pid)
1354 {
1355         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1356         bool max_txq_valid = false;
1357         portid_t pi;
1358         struct rte_eth_dev_info dev_info;
1359
1360         RTE_ETH_FOREACH_DEV(pi) {
1361                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1362                         continue;
1363
1364                 max_txq_valid = true;
1365                 if (dev_info.max_tx_queues < allowed_max_txq) {
1366                         allowed_max_txq = dev_info.max_tx_queues;
1367                         *pid = pi;
1368                 }
1369         }
1370         return max_txq_valid ? allowed_max_txq : 0;
1371 }
1372
1373 /*
1374  * Check input txq is valid or not.
1375  * If input txq is not greater than any of maximum number
1376  * of TX queues of all ports, it is valid.
1377  * if valid, return 0, else return -1
1378  */
1379 int
1380 check_nb_txq(queueid_t txq)
1381 {
1382         queueid_t allowed_max_txq;
1383         portid_t pid = 0;
1384
1385         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1386         if (txq > allowed_max_txq) {
1387                 fprintf(stderr,
1388                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1389                         txq, allowed_max_txq, pid);
1390                 return -1;
1391         }
1392         return 0;
1393 }
1394
1395 /*
1396  * Get the allowed maximum number of RXDs of every rx queue.
1397  * *pid return the port id which has minimal value of
1398  * max_rxd in all queues of all ports.
1399  */
1400 static uint16_t
1401 get_allowed_max_nb_rxd(portid_t *pid)
1402 {
1403         uint16_t allowed_max_rxd = UINT16_MAX;
1404         portid_t pi;
1405         struct rte_eth_dev_info dev_info;
1406
1407         RTE_ETH_FOREACH_DEV(pi) {
1408                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1409                         continue;
1410
1411                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1412                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1413                         *pid = pi;
1414                 }
1415         }
1416         return allowed_max_rxd;
1417 }
1418
1419 /*
1420  * Get the allowed minimal number of RXDs of every rx queue.
1421  * *pid return the port id which has minimal value of
1422  * min_rxd in all queues of all ports.
1423  */
1424 static uint16_t
1425 get_allowed_min_nb_rxd(portid_t *pid)
1426 {
1427         uint16_t allowed_min_rxd = 0;
1428         portid_t pi;
1429         struct rte_eth_dev_info dev_info;
1430
1431         RTE_ETH_FOREACH_DEV(pi) {
1432                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1433                         continue;
1434
1435                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1436                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1437                         *pid = pi;
1438                 }
1439         }
1440
1441         return allowed_min_rxd;
1442 }
1443
1444 /*
1445  * Check input rxd is valid or not.
1446  * If input rxd is not greater than any of maximum number
1447  * of RXDs of every Rx queues and is not less than any of
1448  * minimal number of RXDs of every Rx queues, it is valid.
1449  * if valid, return 0, else return -1
1450  */
1451 int
1452 check_nb_rxd(queueid_t rxd)
1453 {
1454         uint16_t allowed_max_rxd;
1455         uint16_t allowed_min_rxd;
1456         portid_t pid = 0;
1457
1458         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1459         if (rxd > allowed_max_rxd) {
1460                 fprintf(stderr,
1461                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1462                         rxd, allowed_max_rxd, pid);
1463                 return -1;
1464         }
1465
1466         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1467         if (rxd < allowed_min_rxd) {
1468                 fprintf(stderr,
1469                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1470                         rxd, allowed_min_rxd, pid);
1471                 return -1;
1472         }
1473
1474         return 0;
1475 }
1476
1477 /*
1478  * Get the allowed maximum number of TXDs of every rx queues.
1479  * *pid return the port id which has minimal value of
1480  * max_txd in every tx queue.
1481  */
1482 static uint16_t
1483 get_allowed_max_nb_txd(portid_t *pid)
1484 {
1485         uint16_t allowed_max_txd = UINT16_MAX;
1486         portid_t pi;
1487         struct rte_eth_dev_info dev_info;
1488
1489         RTE_ETH_FOREACH_DEV(pi) {
1490                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1491                         continue;
1492
1493                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1494                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1495                         *pid = pi;
1496                 }
1497         }
1498         return allowed_max_txd;
1499 }
1500
1501 /*
1502  * Get the allowed maximum number of TXDs of every tx queues.
1503  * *pid return the port id which has minimal value of
1504  * min_txd in every tx queue.
1505  */
1506 static uint16_t
1507 get_allowed_min_nb_txd(portid_t *pid)
1508 {
1509         uint16_t allowed_min_txd = 0;
1510         portid_t pi;
1511         struct rte_eth_dev_info dev_info;
1512
1513         RTE_ETH_FOREACH_DEV(pi) {
1514                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1515                         continue;
1516
1517                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1518                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1519                         *pid = pi;
1520                 }
1521         }
1522
1523         return allowed_min_txd;
1524 }
1525
1526 /*
1527  * Check input txd is valid or not.
1528  * If input txd is not greater than any of maximum number
1529  * of TXDs of every Rx queues, it is valid.
1530  * if valid, return 0, else return -1
1531  */
1532 int
1533 check_nb_txd(queueid_t txd)
1534 {
1535         uint16_t allowed_max_txd;
1536         uint16_t allowed_min_txd;
1537         portid_t pid = 0;
1538
1539         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1540         if (txd > allowed_max_txd) {
1541                 fprintf(stderr,
1542                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1543                         txd, allowed_max_txd, pid);
1544                 return -1;
1545         }
1546
1547         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1548         if (txd < allowed_min_txd) {
1549                 fprintf(stderr,
1550                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1551                         txd, allowed_min_txd, pid);
1552                 return -1;
1553         }
1554         return 0;
1555 }
1556
1557
1558 /*
1559  * Get the allowed maximum number of hairpin queues.
1560  * *pid return the port id which has minimal value of
1561  * max_hairpin_queues in all ports.
1562  */
1563 queueid_t
1564 get_allowed_max_nb_hairpinq(portid_t *pid)
1565 {
1566         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1567         portid_t pi;
1568         struct rte_eth_hairpin_cap cap;
1569
1570         RTE_ETH_FOREACH_DEV(pi) {
1571                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1572                         *pid = pi;
1573                         return 0;
1574                 }
1575                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1576                         allowed_max_hairpinq = cap.max_nb_queues;
1577                         *pid = pi;
1578                 }
1579         }
1580         return allowed_max_hairpinq;
1581 }
1582
1583 /*
1584  * Check input hairpin is valid or not.
1585  * If input hairpin is not greater than any of maximum number
1586  * of hairpin queues of all ports, it is valid.
1587  * if valid, return 0, else return -1
1588  */
1589 int
1590 check_nb_hairpinq(queueid_t hairpinq)
1591 {
1592         queueid_t allowed_max_hairpinq;
1593         portid_t pid = 0;
1594
1595         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1596         if (hairpinq > allowed_max_hairpinq) {
1597                 fprintf(stderr,
1598                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1599                         hairpinq, allowed_max_hairpinq, pid);
1600                 return -1;
1601         }
1602         return 0;
1603 }
1604
1605 static int
1606 get_eth_overhead(struct rte_eth_dev_info *dev_info)
1607 {
1608         uint32_t eth_overhead;
1609
1610         if (dev_info->max_mtu != UINT16_MAX &&
1611             dev_info->max_rx_pktlen > dev_info->max_mtu)
1612                 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1613         else
1614                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1615
1616         return eth_overhead;
1617 }
1618
1619 static void
1620 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1621 {
1622         struct rte_port *port = &ports[pid];
1623         int ret;
1624         int i;
1625
1626         eth_rx_metadata_negotiate_mp(pid);
1627
1628         port->dev_conf.txmode = tx_mode;
1629         port->dev_conf.rxmode = rx_mode;
1630
1631         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1632         if (ret != 0)
1633                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1634
1635         if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1636                 port->dev_conf.txmode.offloads &=
1637                         ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1638
1639         /* Apply Rx offloads configuration */
1640         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1641                 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads;
1642         /* Apply Tx offloads configuration */
1643         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1644                 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads;
1645
1646         if (eth_link_speed)
1647                 port->dev_conf.link_speeds = eth_link_speed;
1648
1649         if (max_rx_pkt_len)
1650                 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1651                         get_eth_overhead(&port->dev_info);
1652
1653         /* set flag to initialize port/queue */
1654         port->need_reconfig = 1;
1655         port->need_reconfig_queues = 1;
1656         port->socket_id = socket_id;
1657         port->tx_metadata = 0;
1658
1659         /*
1660          * Check for maximum number of segments per MTU.
1661          * Accordingly update the mbuf data size.
1662          */
1663         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1664             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1665                 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1666                 uint16_t mtu;
1667
1668                 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1669                         uint16_t data_size = (mtu + eth_overhead) /
1670                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1671                         uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1672
1673                         if (buffer_size > mbuf_data_size[0]) {
1674                                 mbuf_data_size[0] = buffer_size;
1675                                 TESTPMD_LOG(WARNING,
1676                                         "Configured mbuf size of the first segment %hu\n",
1677                                         mbuf_data_size[0]);
1678                         }
1679                 }
1680         }
1681 }
1682
1683 static void
1684 init_config(void)
1685 {
1686         portid_t pid;
1687         struct rte_mempool *mbp;
1688         unsigned int nb_mbuf_per_pool;
1689         lcoreid_t  lc_id;
1690 #ifdef RTE_LIB_GRO
1691         struct rte_gro_param gro_param;
1692 #endif
1693 #ifdef RTE_LIB_GSO
1694         uint32_t gso_types;
1695 #endif
1696
1697         /* Configuration of logical cores. */
1698         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1699                                 sizeof(struct fwd_lcore *) * nb_lcores,
1700                                 RTE_CACHE_LINE_SIZE);
1701         if (fwd_lcores == NULL) {
1702                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1703                                                         "failed\n", nb_lcores);
1704         }
1705         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1706                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1707                                                sizeof(struct fwd_lcore),
1708                                                RTE_CACHE_LINE_SIZE);
1709                 if (fwd_lcores[lc_id] == NULL) {
1710                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1711                                                                 "failed\n");
1712                 }
1713                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1714         }
1715
1716         RTE_ETH_FOREACH_DEV(pid) {
1717                 uint32_t socket_id;
1718
1719                 if (numa_support) {
1720                         socket_id = port_numa[pid];
1721                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1722                                 socket_id = rte_eth_dev_socket_id(pid);
1723
1724                                 /*
1725                                  * if socket_id is invalid,
1726                                  * set to the first available socket.
1727                                  */
1728                                 if (check_socket_id(socket_id) < 0)
1729                                         socket_id = socket_ids[0];
1730                         }
1731                 } else {
1732                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1733                                     0 : socket_num;
1734                 }
1735                 /* Apply default TxRx configuration for all ports */
1736                 init_config_port_offloads(pid, socket_id);
1737         }
1738         /*
1739          * Create pools of mbuf.
1740          * If NUMA support is disabled, create a single pool of mbuf in
1741          * socket 0 memory by default.
1742          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1743          *
1744          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1745          * nb_txd can be configured at run time.
1746          */
1747         if (param_total_num_mbufs)
1748                 nb_mbuf_per_pool = param_total_num_mbufs;
1749         else {
1750                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1751                         (nb_lcores * mb_mempool_cache) +
1752                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1753                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1754         }
1755
1756         if (numa_support) {
1757                 uint8_t i, j;
1758
1759                 for (i = 0; i < num_sockets; i++)
1760                         for (j = 0; j < mbuf_data_size_n; j++)
1761                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1762                                         mbuf_pool_create(mbuf_data_size[j],
1763                                                           nb_mbuf_per_pool,
1764                                                           socket_ids[i], j);
1765         } else {
1766                 uint8_t i;
1767
1768                 for (i = 0; i < mbuf_data_size_n; i++)
1769                         mempools[i] = mbuf_pool_create
1770                                         (mbuf_data_size[i],
1771                                          nb_mbuf_per_pool,
1772                                          socket_num == UMA_NO_CONFIG ?
1773                                          0 : socket_num, i);
1774         }
1775
1776         init_port_config();
1777
1778 #ifdef RTE_LIB_GSO
1779         gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1780                 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1781 #endif
1782         /*
1783          * Records which Mbuf pool to use by each logical core, if needed.
1784          */
1785         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1786                 mbp = mbuf_pool_find(
1787                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1788
1789                 if (mbp == NULL)
1790                         mbp = mbuf_pool_find(0, 0);
1791                 fwd_lcores[lc_id]->mbp = mbp;
1792 #ifdef RTE_LIB_GSO
1793                 /* initialize GSO context */
1794                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1795                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1796                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1797                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1798                         RTE_ETHER_CRC_LEN;
1799                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1800 #endif
1801         }
1802
1803         fwd_config_setup();
1804
1805 #ifdef RTE_LIB_GRO
1806         /* create a gro context for each lcore */
1807         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1808         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1809         gro_param.max_item_per_flow = MAX_PKT_BURST;
1810         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1811                 gro_param.socket_id = rte_lcore_to_socket_id(
1812                                 fwd_lcores_cpuids[lc_id]);
1813                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1814                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1815                         rte_exit(EXIT_FAILURE,
1816                                         "rte_gro_ctx_create() failed\n");
1817                 }
1818         }
1819 #endif
1820 }
1821
1822
1823 void
1824 reconfig(portid_t new_port_id, unsigned socket_id)
1825 {
1826         /* Reconfiguration of Ethernet ports. */
1827         init_config_port_offloads(new_port_id, socket_id);
1828         init_port_config();
1829 }
1830
1831 int
1832 init_fwd_streams(void)
1833 {
1834         portid_t pid;
1835         struct rte_port *port;
1836         streamid_t sm_id, nb_fwd_streams_new;
1837         queueid_t q;
1838
1839         /* set socket id according to numa or not */
1840         RTE_ETH_FOREACH_DEV(pid) {
1841                 port = &ports[pid];
1842                 if (nb_rxq > port->dev_info.max_rx_queues) {
1843                         fprintf(stderr,
1844                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1845                                 nb_rxq, port->dev_info.max_rx_queues);
1846                         return -1;
1847                 }
1848                 if (nb_txq > port->dev_info.max_tx_queues) {
1849                         fprintf(stderr,
1850                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1851                                 nb_txq, port->dev_info.max_tx_queues);
1852                         return -1;
1853                 }
1854                 if (numa_support) {
1855                         if (port_numa[pid] != NUMA_NO_CONFIG)
1856                                 port->socket_id = port_numa[pid];
1857                         else {
1858                                 port->socket_id = rte_eth_dev_socket_id(pid);
1859
1860                                 /*
1861                                  * if socket_id is invalid,
1862                                  * set to the first available socket.
1863                                  */
1864                                 if (check_socket_id(port->socket_id) < 0)
1865                                         port->socket_id = socket_ids[0];
1866                         }
1867                 }
1868                 else {
1869                         if (socket_num == UMA_NO_CONFIG)
1870                                 port->socket_id = 0;
1871                         else
1872                                 port->socket_id = socket_num;
1873                 }
1874         }
1875
1876         q = RTE_MAX(nb_rxq, nb_txq);
1877         if (q == 0) {
1878                 fprintf(stderr,
1879                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1880                 return -1;
1881         }
1882         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1883         if (nb_fwd_streams_new == nb_fwd_streams)
1884                 return 0;
1885         /* clear the old */
1886         if (fwd_streams != NULL) {
1887                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1888                         if (fwd_streams[sm_id] == NULL)
1889                                 continue;
1890                         rte_free(fwd_streams[sm_id]);
1891                         fwd_streams[sm_id] = NULL;
1892                 }
1893                 rte_free(fwd_streams);
1894                 fwd_streams = NULL;
1895         }
1896
1897         /* init new */
1898         nb_fwd_streams = nb_fwd_streams_new;
1899         if (nb_fwd_streams) {
1900                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1901                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1902                         RTE_CACHE_LINE_SIZE);
1903                 if (fwd_streams == NULL)
1904                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1905                                  " (struct fwd_stream *)) failed\n",
1906                                  nb_fwd_streams);
1907
1908                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1909                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1910                                 " struct fwd_stream", sizeof(struct fwd_stream),
1911                                 RTE_CACHE_LINE_SIZE);
1912                         if (fwd_streams[sm_id] == NULL)
1913                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1914                                          "(struct fwd_stream) failed\n");
1915                 }
1916         }
1917
1918         return 0;
1919 }
1920
1921 static void
1922 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1923 {
1924         uint64_t total_burst, sburst;
1925         uint64_t nb_burst;
1926         uint64_t burst_stats[4];
1927         uint16_t pktnb_stats[4];
1928         uint16_t nb_pkt;
1929         int burst_percent[4], sburstp;
1930         int i;
1931
1932         /*
1933          * First compute the total number of packet bursts and the
1934          * two highest numbers of bursts of the same number of packets.
1935          */
1936         memset(&burst_stats, 0x0, sizeof(burst_stats));
1937         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1938
1939         /* Show stats for 0 burst size always */
1940         total_burst = pbs->pkt_burst_spread[0];
1941         burst_stats[0] = pbs->pkt_burst_spread[0];
1942         pktnb_stats[0] = 0;
1943
1944         /* Find the next 2 burst sizes with highest occurrences. */
1945         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1946                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1947
1948                 if (nb_burst == 0)
1949                         continue;
1950
1951                 total_burst += nb_burst;
1952
1953                 if (nb_burst > burst_stats[1]) {
1954                         burst_stats[2] = burst_stats[1];
1955                         pktnb_stats[2] = pktnb_stats[1];
1956                         burst_stats[1] = nb_burst;
1957                         pktnb_stats[1] = nb_pkt;
1958                 } else if (nb_burst > burst_stats[2]) {
1959                         burst_stats[2] = nb_burst;
1960                         pktnb_stats[2] = nb_pkt;
1961                 }
1962         }
1963         if (total_burst == 0)
1964                 return;
1965
1966         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1967         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1968                 if (i == 3) {
1969                         printf("%d%% of other]\n", 100 - sburstp);
1970                         return;
1971                 }
1972
1973                 sburst += burst_stats[i];
1974                 if (sburst == total_burst) {
1975                         printf("%d%% of %d pkts]\n",
1976                                 100 - sburstp, (int) pktnb_stats[i]);
1977                         return;
1978                 }
1979
1980                 burst_percent[i] =
1981                         (double)burst_stats[i] / total_burst * 100;
1982                 printf("%d%% of %d pkts + ",
1983                         burst_percent[i], (int) pktnb_stats[i]);
1984                 sburstp += burst_percent[i];
1985         }
1986 }
1987
1988 static void
1989 fwd_stream_stats_display(streamid_t stream_id)
1990 {
1991         struct fwd_stream *fs;
1992         static const char *fwd_top_stats_border = "-------";
1993
1994         fs = fwd_streams[stream_id];
1995         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1996             (fs->fwd_dropped == 0))
1997                 return;
1998         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1999                "TX Port=%2d/Queue=%2d %s\n",
2000                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
2001                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
2002         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
2003                " TX-dropped: %-14"PRIu64,
2004                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
2005
2006         /* if checksum mode */
2007         if (cur_fwd_eng == &csum_fwd_engine) {
2008                 printf("  RX- bad IP checksum: %-14"PRIu64
2009                        "  Rx- bad L4 checksum: %-14"PRIu64
2010                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
2011                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
2012                         fs->rx_bad_outer_l4_csum);
2013                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
2014                         fs->rx_bad_outer_ip_csum);
2015         } else {
2016                 printf("\n");
2017         }
2018
2019         if (record_burst_stats) {
2020                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
2021                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
2022         }
2023 }
2024
2025 void
2026 fwd_stats_display(void)
2027 {
2028         static const char *fwd_stats_border = "----------------------";
2029         static const char *acc_stats_border = "+++++++++++++++";
2030         struct {
2031                 struct fwd_stream *rx_stream;
2032                 struct fwd_stream *tx_stream;
2033                 uint64_t tx_dropped;
2034                 uint64_t rx_bad_ip_csum;
2035                 uint64_t rx_bad_l4_csum;
2036                 uint64_t rx_bad_outer_l4_csum;
2037                 uint64_t rx_bad_outer_ip_csum;
2038         } ports_stats[RTE_MAX_ETHPORTS];
2039         uint64_t total_rx_dropped = 0;
2040         uint64_t total_tx_dropped = 0;
2041         uint64_t total_rx_nombuf = 0;
2042         struct rte_eth_stats stats;
2043         uint64_t fwd_cycles = 0;
2044         uint64_t total_recv = 0;
2045         uint64_t total_xmit = 0;
2046         struct rte_port *port;
2047         streamid_t sm_id;
2048         portid_t pt_id;
2049         int ret;
2050         int i;
2051
2052         memset(ports_stats, 0, sizeof(ports_stats));
2053
2054         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2055                 struct fwd_stream *fs = fwd_streams[sm_id];
2056
2057                 if (cur_fwd_config.nb_fwd_streams >
2058                     cur_fwd_config.nb_fwd_ports) {
2059                         fwd_stream_stats_display(sm_id);
2060                 } else {
2061                         ports_stats[fs->tx_port].tx_stream = fs;
2062                         ports_stats[fs->rx_port].rx_stream = fs;
2063                 }
2064
2065                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
2066
2067                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
2068                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2069                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2070                                 fs->rx_bad_outer_l4_csum;
2071                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2072                                 fs->rx_bad_outer_ip_csum;
2073
2074                 if (record_core_cycles)
2075                         fwd_cycles += fs->core_cycles;
2076         }
2077         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2078                 pt_id = fwd_ports_ids[i];
2079                 port = &ports[pt_id];
2080
2081                 ret = rte_eth_stats_get(pt_id, &stats);
2082                 if (ret != 0) {
2083                         fprintf(stderr,
2084                                 "%s: Error: failed to get stats (port %u): %d",
2085                                 __func__, pt_id, ret);
2086                         continue;
2087                 }
2088                 stats.ipackets -= port->stats.ipackets;
2089                 stats.opackets -= port->stats.opackets;
2090                 stats.ibytes -= port->stats.ibytes;
2091                 stats.obytes -= port->stats.obytes;
2092                 stats.imissed -= port->stats.imissed;
2093                 stats.oerrors -= port->stats.oerrors;
2094                 stats.rx_nombuf -= port->stats.rx_nombuf;
2095
2096                 total_recv += stats.ipackets;
2097                 total_xmit += stats.opackets;
2098                 total_rx_dropped += stats.imissed;
2099                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2100                 total_tx_dropped += stats.oerrors;
2101                 total_rx_nombuf  += stats.rx_nombuf;
2102
2103                 printf("\n  %s Forward statistics for port %-2d %s\n",
2104                        fwd_stats_border, pt_id, fwd_stats_border);
2105
2106                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2107                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2108                        stats.ipackets + stats.imissed);
2109
2110                 if (cur_fwd_eng == &csum_fwd_engine) {
2111                         printf("  Bad-ipcsum: %-14"PRIu64
2112                                " Bad-l4csum: %-14"PRIu64
2113                                "Bad-outer-l4csum: %-14"PRIu64"\n",
2114                                ports_stats[pt_id].rx_bad_ip_csum,
2115                                ports_stats[pt_id].rx_bad_l4_csum,
2116                                ports_stats[pt_id].rx_bad_outer_l4_csum);
2117                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
2118                                ports_stats[pt_id].rx_bad_outer_ip_csum);
2119                 }
2120                 if (stats.ierrors + stats.rx_nombuf > 0) {
2121                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
2122                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2123                 }
2124
2125                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2126                        "TX-total: %-"PRIu64"\n",
2127                        stats.opackets, ports_stats[pt_id].tx_dropped,
2128                        stats.opackets + ports_stats[pt_id].tx_dropped);
2129
2130                 if (record_burst_stats) {
2131                         if (ports_stats[pt_id].rx_stream)
2132                                 pkt_burst_stats_display("RX",
2133                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
2134                         if (ports_stats[pt_id].tx_stream)
2135                                 pkt_burst_stats_display("TX",
2136                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2137                 }
2138
2139                 printf("  %s--------------------------------%s\n",
2140                        fwd_stats_border, fwd_stats_border);
2141         }
2142
2143         printf("\n  %s Accumulated forward statistics for all ports"
2144                "%s\n",
2145                acc_stats_border, acc_stats_border);
2146         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2147                "%-"PRIu64"\n"
2148                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2149                "%-"PRIu64"\n",
2150                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2151                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2152         if (total_rx_nombuf > 0)
2153                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2154         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
2155                "%s\n",
2156                acc_stats_border, acc_stats_border);
2157         if (record_core_cycles) {
2158 #define CYC_PER_MHZ 1E6
2159                 if (total_recv > 0 || total_xmit > 0) {
2160                         uint64_t total_pkts = 0;
2161                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2162                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2163                                 total_pkts = total_xmit;
2164                         else
2165                                 total_pkts = total_recv;
2166
2167                         printf("\n  CPU cycles/packet=%.2F (total cycles="
2168                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2169                                " MHz Clock\n",
2170                                (double) fwd_cycles / total_pkts,
2171                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2172                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2173                 }
2174         }
2175 }
2176
2177 void
2178 fwd_stats_reset(void)
2179 {
2180         streamid_t sm_id;
2181         portid_t pt_id;
2182         int ret;
2183         int i;
2184
2185         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2186                 pt_id = fwd_ports_ids[i];
2187                 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2188                 if (ret != 0)
2189                         fprintf(stderr,
2190                                 "%s: Error: failed to clear stats (port %u):%d",
2191                                 __func__, pt_id, ret);
2192         }
2193         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2194                 struct fwd_stream *fs = fwd_streams[sm_id];
2195
2196                 fs->rx_packets = 0;
2197                 fs->tx_packets = 0;
2198                 fs->fwd_dropped = 0;
2199                 fs->rx_bad_ip_csum = 0;
2200                 fs->rx_bad_l4_csum = 0;
2201                 fs->rx_bad_outer_l4_csum = 0;
2202                 fs->rx_bad_outer_ip_csum = 0;
2203
2204                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2205                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2206                 fs->core_cycles = 0;
2207         }
2208 }
2209
2210 static void
2211 flush_fwd_rx_queues(void)
2212 {
2213         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2214         portid_t  rxp;
2215         portid_t port_id;
2216         queueid_t rxq;
2217         uint16_t  nb_rx;
2218         uint16_t  i;
2219         uint8_t   j;
2220         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2221         uint64_t timer_period;
2222
2223         if (num_procs > 1) {
2224                 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2225                 return;
2226         }
2227
2228         /* convert to number of cycles */
2229         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2230
2231         for (j = 0; j < 2; j++) {
2232                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2233                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2234                                 port_id = fwd_ports_ids[rxp];
2235
2236                                 /* Polling stopped queues is prohibited. */
2237                                 if (ports[port_id].rxq[rxq].state ==
2238                                     RTE_ETH_QUEUE_STATE_STOPPED)
2239                                         continue;
2240
2241                                 /**
2242                                 * testpmd can stuck in the below do while loop
2243                                 * if rte_eth_rx_burst() always returns nonzero
2244                                 * packets. So timer is added to exit this loop
2245                                 * after 1sec timer expiry.
2246                                 */
2247                                 prev_tsc = rte_rdtsc();
2248                                 do {
2249                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2250                                                 pkts_burst, MAX_PKT_BURST);
2251                                         for (i = 0; i < nb_rx; i++)
2252                                                 rte_pktmbuf_free(pkts_burst[i]);
2253
2254                                         cur_tsc = rte_rdtsc();
2255                                         diff_tsc = cur_tsc - prev_tsc;
2256                                         timer_tsc += diff_tsc;
2257                                 } while ((nb_rx > 0) &&
2258                                         (timer_tsc < timer_period));
2259                                 timer_tsc = 0;
2260                         }
2261                 }
2262                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2263         }
2264 }
2265
2266 static void
2267 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2268 {
2269         struct fwd_stream **fsm;
2270         streamid_t nb_fs;
2271         streamid_t sm_id;
2272 #ifdef RTE_LIB_BITRATESTATS
2273         uint64_t tics_per_1sec;
2274         uint64_t tics_datum;
2275         uint64_t tics_current;
2276         uint16_t i, cnt_ports;
2277
2278         cnt_ports = nb_ports;
2279         tics_datum = rte_rdtsc();
2280         tics_per_1sec = rte_get_timer_hz();
2281 #endif
2282         fsm = &fwd_streams[fc->stream_idx];
2283         nb_fs = fc->stream_nb;
2284         do {
2285                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2286                         if (!fsm[sm_id]->disabled)
2287                                 (*pkt_fwd)(fsm[sm_id]);
2288 #ifdef RTE_LIB_BITRATESTATS
2289                 if (bitrate_enabled != 0 &&
2290                                 bitrate_lcore_id == rte_lcore_id()) {
2291                         tics_current = rte_rdtsc();
2292                         if (tics_current - tics_datum >= tics_per_1sec) {
2293                                 /* Periodic bitrate calculation */
2294                                 for (i = 0; i < cnt_ports; i++)
2295                                         rte_stats_bitrate_calc(bitrate_data,
2296                                                 ports_ids[i]);
2297                                 tics_datum = tics_current;
2298                         }
2299                 }
2300 #endif
2301 #ifdef RTE_LIB_LATENCYSTATS
2302                 if (latencystats_enabled != 0 &&
2303                                 latencystats_lcore_id == rte_lcore_id())
2304                         rte_latencystats_update();
2305 #endif
2306
2307         } while (! fc->stopped);
2308 }
2309
2310 static int
2311 start_pkt_forward_on_core(void *fwd_arg)
2312 {
2313         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2314                              cur_fwd_config.fwd_eng->packet_fwd);
2315         return 0;
2316 }
2317
2318 /*
2319  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2320  * Used to start communication flows in network loopback test configurations.
2321  */
2322 static int
2323 run_one_txonly_burst_on_core(void *fwd_arg)
2324 {
2325         struct fwd_lcore *fwd_lc;
2326         struct fwd_lcore tmp_lcore;
2327
2328         fwd_lc = (struct fwd_lcore *) fwd_arg;
2329         tmp_lcore = *fwd_lc;
2330         tmp_lcore.stopped = 1;
2331         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2332         return 0;
2333 }
2334
2335 /*
2336  * Launch packet forwarding:
2337  *     - Setup per-port forwarding context.
2338  *     - launch logical cores with their forwarding configuration.
2339  */
2340 static void
2341 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2342 {
2343         unsigned int i;
2344         unsigned int lc_id;
2345         int diag;
2346
2347         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2348                 lc_id = fwd_lcores_cpuids[i];
2349                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2350                         fwd_lcores[i]->stopped = 0;
2351                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2352                                                      fwd_lcores[i], lc_id);
2353                         if (diag != 0)
2354                                 fprintf(stderr,
2355                                         "launch lcore %u failed - diag=%d\n",
2356                                         lc_id, diag);
2357                 }
2358         }
2359 }
2360
2361 /*
2362  * Launch packet forwarding configuration.
2363  */
2364 void
2365 start_packet_forwarding(int with_tx_first)
2366 {
2367         port_fwd_begin_t port_fwd_begin;
2368         port_fwd_end_t  port_fwd_end;
2369         stream_init_t stream_init = cur_fwd_eng->stream_init;
2370         unsigned int i;
2371
2372         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2373                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2374
2375         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2376                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2377
2378         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2379                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2380                 (!nb_rxq || !nb_txq))
2381                 rte_exit(EXIT_FAILURE,
2382                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2383                         cur_fwd_eng->fwd_mode_name);
2384
2385         if (all_ports_started() == 0) {
2386                 fprintf(stderr, "Not all ports were started\n");
2387                 return;
2388         }
2389         if (test_done == 0) {
2390                 fprintf(stderr, "Packet forwarding already started\n");
2391                 return;
2392         }
2393
2394         fwd_config_setup();
2395
2396         pkt_fwd_config_display(&cur_fwd_config);
2397         if (!pkt_fwd_shared_rxq_check())
2398                 return;
2399
2400         if (stream_init != NULL)
2401                 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++)
2402                         stream_init(fwd_streams[i]);
2403
2404         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2405         if (port_fwd_begin != NULL) {
2406                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2407                         if (port_fwd_begin(fwd_ports_ids[i])) {
2408                                 fprintf(stderr,
2409                                         "Packet forwarding is not ready\n");
2410                                 return;
2411                         }
2412                 }
2413         }
2414
2415         if (with_tx_first) {
2416                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2417                 if (port_fwd_begin != NULL) {
2418                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2419                                 if (port_fwd_begin(fwd_ports_ids[i])) {
2420                                         fprintf(stderr,
2421                                                 "Packet forwarding is not ready\n");
2422                                         return;
2423                                 }
2424                         }
2425                 }
2426         }
2427
2428         test_done = 0;
2429
2430         if(!no_flush_rx)
2431                 flush_fwd_rx_queues();
2432
2433         rxtx_config_display();
2434
2435         fwd_stats_reset();
2436         if (with_tx_first) {
2437                 while (with_tx_first--) {
2438                         launch_packet_forwarding(
2439                                         run_one_txonly_burst_on_core);
2440                         rte_eal_mp_wait_lcore();
2441                 }
2442                 port_fwd_end = tx_only_engine.port_fwd_end;
2443                 if (port_fwd_end != NULL) {
2444                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2445                                 (*port_fwd_end)(fwd_ports_ids[i]);
2446                 }
2447         }
2448         launch_packet_forwarding(start_pkt_forward_on_core);
2449 }
2450
2451 void
2452 stop_packet_forwarding(void)
2453 {
2454         port_fwd_end_t port_fwd_end;
2455         lcoreid_t lc_id;
2456         portid_t pt_id;
2457         int i;
2458
2459         if (test_done) {
2460                 fprintf(stderr, "Packet forwarding not started\n");
2461                 return;
2462         }
2463         printf("Telling cores to stop...");
2464         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2465                 fwd_lcores[lc_id]->stopped = 1;
2466         printf("\nWaiting for lcores to finish...\n");
2467         rte_eal_mp_wait_lcore();
2468         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2469         if (port_fwd_end != NULL) {
2470                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2471                         pt_id = fwd_ports_ids[i];
2472                         (*port_fwd_end)(pt_id);
2473                 }
2474         }
2475
2476         fwd_stats_display();
2477
2478         printf("\nDone.\n");
2479         test_done = 1;
2480 }
2481
2482 void
2483 dev_set_link_up(portid_t pid)
2484 {
2485         if (rte_eth_dev_set_link_up(pid) < 0)
2486                 fprintf(stderr, "\nSet link up fail.\n");
2487 }
2488
2489 void
2490 dev_set_link_down(portid_t pid)
2491 {
2492         if (rte_eth_dev_set_link_down(pid) < 0)
2493                 fprintf(stderr, "\nSet link down fail.\n");
2494 }
2495
2496 static int
2497 all_ports_started(void)
2498 {
2499         portid_t pi;
2500         struct rte_port *port;
2501
2502         RTE_ETH_FOREACH_DEV(pi) {
2503                 port = &ports[pi];
2504                 /* Check if there is a port which is not started */
2505                 if ((port->port_status != RTE_PORT_STARTED) &&
2506                         (port->slave_flag == 0))
2507                         return 0;
2508         }
2509
2510         /* No port is not started */
2511         return 1;
2512 }
2513
2514 int
2515 port_is_stopped(portid_t port_id)
2516 {
2517         struct rte_port *port = &ports[port_id];
2518
2519         if ((port->port_status != RTE_PORT_STOPPED) &&
2520             (port->slave_flag == 0))
2521                 return 0;
2522         return 1;
2523 }
2524
2525 int
2526 all_ports_stopped(void)
2527 {
2528         portid_t pi;
2529
2530         RTE_ETH_FOREACH_DEV(pi) {
2531                 if (!port_is_stopped(pi))
2532                         return 0;
2533         }
2534
2535         return 1;
2536 }
2537
2538 int
2539 port_is_started(portid_t port_id)
2540 {
2541         if (port_id_is_invalid(port_id, ENABLED_WARN))
2542                 return 0;
2543
2544         if (ports[port_id].port_status != RTE_PORT_STARTED)
2545                 return 0;
2546
2547         return 1;
2548 }
2549
2550 /* Configure the Rx and Tx hairpin queues for the selected port. */
2551 static int
2552 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2553 {
2554         queueid_t qi;
2555         struct rte_eth_hairpin_conf hairpin_conf = {
2556                 .peer_count = 1,
2557         };
2558         int i;
2559         int diag;
2560         struct rte_port *port = &ports[pi];
2561         uint16_t peer_rx_port = pi;
2562         uint16_t peer_tx_port = pi;
2563         uint32_t manual = 1;
2564         uint32_t tx_exp = hairpin_mode & 0x10;
2565
2566         if (!(hairpin_mode & 0xf)) {
2567                 peer_rx_port = pi;
2568                 peer_tx_port = pi;
2569                 manual = 0;
2570         } else if (hairpin_mode & 0x1) {
2571                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2572                                                        RTE_ETH_DEV_NO_OWNER);
2573                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2574                         peer_tx_port = rte_eth_find_next_owned_by(0,
2575                                                 RTE_ETH_DEV_NO_OWNER);
2576                 if (p_pi != RTE_MAX_ETHPORTS) {
2577                         peer_rx_port = p_pi;
2578                 } else {
2579                         uint16_t next_pi;
2580
2581                         /* Last port will be the peer RX port of the first. */
2582                         RTE_ETH_FOREACH_DEV(next_pi)
2583                                 peer_rx_port = next_pi;
2584                 }
2585                 manual = 1;
2586         } else if (hairpin_mode & 0x2) {
2587                 if (cnt_pi & 0x1) {
2588                         peer_rx_port = p_pi;
2589                 } else {
2590                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2591                                                 RTE_ETH_DEV_NO_OWNER);
2592                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2593                                 peer_rx_port = pi;
2594                 }
2595                 peer_tx_port = peer_rx_port;
2596                 manual = 1;
2597         }
2598
2599         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2600                 hairpin_conf.peers[0].port = peer_rx_port;
2601                 hairpin_conf.peers[0].queue = i + nb_rxq;
2602                 hairpin_conf.manual_bind = !!manual;
2603                 hairpin_conf.tx_explicit = !!tx_exp;
2604                 diag = rte_eth_tx_hairpin_queue_setup
2605                         (pi, qi, nb_txd, &hairpin_conf);
2606                 i++;
2607                 if (diag == 0)
2608                         continue;
2609
2610                 /* Fail to setup rx queue, return */
2611                 if (port->port_status == RTE_PORT_HANDLING)
2612                         port->port_status = RTE_PORT_STOPPED;
2613                 else
2614                         fprintf(stderr,
2615                                 "Port %d can not be set back to stopped\n", pi);
2616                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2617                         pi);
2618                 /* try to reconfigure queues next time */
2619                 port->need_reconfig_queues = 1;
2620                 return -1;
2621         }
2622         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2623                 hairpin_conf.peers[0].port = peer_tx_port;
2624                 hairpin_conf.peers[0].queue = i + nb_txq;
2625                 hairpin_conf.manual_bind = !!manual;
2626                 hairpin_conf.tx_explicit = !!tx_exp;
2627                 diag = rte_eth_rx_hairpin_queue_setup
2628                         (pi, qi, nb_rxd, &hairpin_conf);
2629                 i++;
2630                 if (diag == 0)
2631                         continue;
2632
2633                 /* Fail to setup rx queue, return */
2634                 if (port->port_status == RTE_PORT_HANDLING)
2635                         port->port_status = RTE_PORT_STOPPED;
2636                 else
2637                         fprintf(stderr,
2638                                 "Port %d can not be set back to stopped\n", pi);
2639                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2640                         pi);
2641                 /* try to reconfigure queues next time */
2642                 port->need_reconfig_queues = 1;
2643                 return -1;
2644         }
2645         return 0;
2646 }
2647
2648 /* Configure the Rx with optional split. */
2649 int
2650 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2651                uint16_t nb_rx_desc, unsigned int socket_id,
2652                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2653 {
2654         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2655         unsigned int i, mp_n;
2656         int ret;
2657
2658         if (rx_pkt_nb_segs <= 1 ||
2659             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2660                 rx_conf->rx_seg = NULL;
2661                 rx_conf->rx_nseg = 0;
2662                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2663                                              nb_rx_desc, socket_id,
2664                                              rx_conf, mp);
2665                 goto exit;
2666         }
2667         for (i = 0; i < rx_pkt_nb_segs; i++) {
2668                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2669                 struct rte_mempool *mpx;
2670                 /*
2671                  * Use last valid pool for the segments with number
2672                  * exceeding the pool index.
2673                  */
2674                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2675                 mpx = mbuf_pool_find(socket_id, mp_n);
2676                 /* Handle zero as mbuf data buffer size. */
2677                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2678                                    rx_pkt_seg_lengths[i] :
2679                                    mbuf_data_size[mp_n];
2680                 rx_seg->offset = i < rx_pkt_nb_offs ?
2681                                    rx_pkt_seg_offsets[i] : 0;
2682                 rx_seg->mp = mpx ? mpx : mp;
2683         }
2684         rx_conf->rx_nseg = rx_pkt_nb_segs;
2685         rx_conf->rx_seg = rx_useg;
2686         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2687                                     socket_id, rx_conf, NULL);
2688         rx_conf->rx_seg = NULL;
2689         rx_conf->rx_nseg = 0;
2690 exit:
2691         ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ?
2692                                                 RTE_ETH_QUEUE_STATE_STOPPED :
2693                                                 RTE_ETH_QUEUE_STATE_STARTED;
2694         return ret;
2695 }
2696
2697 static int
2698 alloc_xstats_display_info(portid_t pi)
2699 {
2700         uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2701         uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2702         uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2703
2704         if (xstats_display_num == 0)
2705                 return 0;
2706
2707         *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2708         if (*ids_supp == NULL)
2709                 goto fail_ids_supp;
2710
2711         *prev_values = calloc(xstats_display_num,
2712                               sizeof(**prev_values));
2713         if (*prev_values == NULL)
2714                 goto fail_prev_values;
2715
2716         *curr_values = calloc(xstats_display_num,
2717                               sizeof(**curr_values));
2718         if (*curr_values == NULL)
2719                 goto fail_curr_values;
2720
2721         ports[pi].xstats_info.allocated = true;
2722
2723         return 0;
2724
2725 fail_curr_values:
2726         free(*prev_values);
2727 fail_prev_values:
2728         free(*ids_supp);
2729 fail_ids_supp:
2730         return -ENOMEM;
2731 }
2732
2733 static void
2734 free_xstats_display_info(portid_t pi)
2735 {
2736         if (!ports[pi].xstats_info.allocated)
2737                 return;
2738         free(ports[pi].xstats_info.ids_supp);
2739         free(ports[pi].xstats_info.prev_values);
2740         free(ports[pi].xstats_info.curr_values);
2741         ports[pi].xstats_info.allocated = false;
2742 }
2743
2744 /** Fill helper structures for specified port to show extended statistics. */
2745 static void
2746 fill_xstats_display_info_for_port(portid_t pi)
2747 {
2748         unsigned int stat, stat_supp;
2749         const char *xstat_name;
2750         struct rte_port *port;
2751         uint64_t *ids_supp;
2752         int rc;
2753
2754         if (xstats_display_num == 0)
2755                 return;
2756
2757         if (pi == (portid_t)RTE_PORT_ALL) {
2758                 fill_xstats_display_info();
2759                 return;
2760         }
2761
2762         port = &ports[pi];
2763         if (port->port_status != RTE_PORT_STARTED)
2764                 return;
2765
2766         if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2767                 rte_exit(EXIT_FAILURE,
2768                          "Failed to allocate xstats display memory\n");
2769
2770         ids_supp = port->xstats_info.ids_supp;
2771         for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2772                 xstat_name = xstats_display[stat].name;
2773                 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2774                                                    ids_supp + stat_supp);
2775                 if (rc != 0) {
2776                         fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2777                                 xstat_name, pi, stat);
2778                         continue;
2779                 }
2780                 stat_supp++;
2781         }
2782
2783         port->xstats_info.ids_supp_sz = stat_supp;
2784 }
2785
2786 /** Fill helper structures for all ports to show extended statistics. */
2787 static void
2788 fill_xstats_display_info(void)
2789 {
2790         portid_t pi;
2791
2792         if (xstats_display_num == 0)
2793                 return;
2794
2795         RTE_ETH_FOREACH_DEV(pi)
2796                 fill_xstats_display_info_for_port(pi);
2797 }
2798
2799 int
2800 start_port(portid_t pid)
2801 {
2802         int diag, need_check_link_status = -1;
2803         portid_t pi;
2804         portid_t p_pi = RTE_MAX_ETHPORTS;
2805         portid_t pl[RTE_MAX_ETHPORTS];
2806         portid_t peer_pl[RTE_MAX_ETHPORTS];
2807         uint16_t cnt_pi = 0;
2808         uint16_t cfg_pi = 0;
2809         int peer_pi;
2810         queueid_t qi;
2811         struct rte_port *port;
2812         struct rte_eth_hairpin_cap cap;
2813
2814         if (port_id_is_invalid(pid, ENABLED_WARN))
2815                 return 0;
2816
2817         RTE_ETH_FOREACH_DEV(pi) {
2818                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2819                         continue;
2820
2821                 if (port_is_bonding_slave(pi)) {
2822                         fprintf(stderr,
2823                                 "Please remove port %d from bonded device.\n",
2824                                 pi);
2825                         continue;
2826                 }
2827
2828                 need_check_link_status = 0;
2829                 port = &ports[pi];
2830                 if (port->port_status == RTE_PORT_STOPPED)
2831                         port->port_status = RTE_PORT_HANDLING;
2832                 else {
2833                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2834                         continue;
2835                 }
2836
2837                 if (port->need_reconfig > 0) {
2838                         struct rte_eth_conf dev_conf;
2839                         int k;
2840
2841                         port->need_reconfig = 0;
2842
2843                         if (flow_isolate_all) {
2844                                 int ret = port_flow_isolate(pi, 1);
2845                                 if (ret) {
2846                                         fprintf(stderr,
2847                                                 "Failed to apply isolated mode on port %d\n",
2848                                                 pi);
2849                                         return -1;
2850                                 }
2851                         }
2852                         configure_rxtx_dump_callbacks(0);
2853                         printf("Configuring Port %d (socket %u)\n", pi,
2854                                         port->socket_id);
2855                         if (nb_hairpinq > 0 &&
2856                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2857                                 fprintf(stderr,
2858                                         "Port %d doesn't support hairpin queues\n",
2859                                         pi);
2860                                 return -1;
2861                         }
2862
2863                         /* configure port */
2864                         diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2865                                                      nb_txq + nb_hairpinq,
2866                                                      &(port->dev_conf));
2867                         if (diag != 0) {
2868                                 if (port->port_status == RTE_PORT_HANDLING)
2869                                         port->port_status = RTE_PORT_STOPPED;
2870                                 else
2871                                         fprintf(stderr,
2872                                                 "Port %d can not be set back to stopped\n",
2873                                                 pi);
2874                                 fprintf(stderr, "Fail to configure port %d\n",
2875                                         pi);
2876                                 /* try to reconfigure port next time */
2877                                 port->need_reconfig = 1;
2878                                 return -1;
2879                         }
2880                         /* get device configuration*/
2881                         if (0 !=
2882                                 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2883                                 fprintf(stderr,
2884                                         "port %d can not get device configuration\n",
2885                                         pi);
2886                                 return -1;
2887                         }
2888                         /* Apply Rx offloads configuration */
2889                         if (dev_conf.rxmode.offloads !=
2890                             port->dev_conf.rxmode.offloads) {
2891                                 port->dev_conf.rxmode.offloads |=
2892                                         dev_conf.rxmode.offloads;
2893                                 for (k = 0;
2894                                      k < port->dev_info.max_rx_queues;
2895                                      k++)
2896                                         port->rxq[k].conf.offloads |=
2897                                                 dev_conf.rxmode.offloads;
2898                         }
2899                         /* Apply Tx offloads configuration */
2900                         if (dev_conf.txmode.offloads !=
2901                             port->dev_conf.txmode.offloads) {
2902                                 port->dev_conf.txmode.offloads |=
2903                                         dev_conf.txmode.offloads;
2904                                 for (k = 0;
2905                                      k < port->dev_info.max_tx_queues;
2906                                      k++)
2907                                         port->txq[k].conf.offloads |=
2908                                                 dev_conf.txmode.offloads;
2909                         }
2910                 }
2911                 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2912                         port->need_reconfig_queues = 0;
2913                         /* setup tx queues */
2914                         for (qi = 0; qi < nb_txq; qi++) {
2915                                 struct rte_eth_txconf *conf =
2916                                                         &port->txq[qi].conf;
2917
2918                                 if ((numa_support) &&
2919                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2920                                         diag = rte_eth_tx_queue_setup(pi, qi,
2921                                                 port->nb_tx_desc[qi],
2922                                                 txring_numa[pi],
2923                                                 &(port->txq[qi].conf));
2924                                 else
2925                                         diag = rte_eth_tx_queue_setup(pi, qi,
2926                                                 port->nb_tx_desc[qi],
2927                                                 port->socket_id,
2928                                                 &(port->txq[qi].conf));
2929
2930                                 if (diag == 0) {
2931                                         port->txq[qi].state =
2932                                                 conf->tx_deferred_start ?
2933                                                 RTE_ETH_QUEUE_STATE_STOPPED :
2934                                                 RTE_ETH_QUEUE_STATE_STARTED;
2935                                         continue;
2936                                 }
2937
2938                                 /* Fail to setup tx queue, return */
2939                                 if (port->port_status == RTE_PORT_HANDLING)
2940                                         port->port_status = RTE_PORT_STOPPED;
2941                                 else
2942                                         fprintf(stderr,
2943                                                 "Port %d can not be set back to stopped\n",
2944                                                 pi);
2945                                 fprintf(stderr,
2946                                         "Fail to configure port %d tx queues\n",
2947                                         pi);
2948                                 /* try to reconfigure queues next time */
2949                                 port->need_reconfig_queues = 1;
2950                                 return -1;
2951                         }
2952                         for (qi = 0; qi < nb_rxq; qi++) {
2953                                 /* setup rx queues */
2954                                 if ((numa_support) &&
2955                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2956                                         struct rte_mempool * mp =
2957                                                 mbuf_pool_find
2958                                                         (rxring_numa[pi], 0);
2959                                         if (mp == NULL) {
2960                                                 fprintf(stderr,
2961                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2962                                                         rxring_numa[pi]);
2963                                                 return -1;
2964                                         }
2965
2966                                         diag = rx_queue_setup(pi, qi,
2967                                              port->nb_rx_desc[qi],
2968                                              rxring_numa[pi],
2969                                              &(port->rxq[qi].conf),
2970                                              mp);
2971                                 } else {
2972                                         struct rte_mempool *mp =
2973                                                 mbuf_pool_find
2974                                                         (port->socket_id, 0);
2975                                         if (mp == NULL) {
2976                                                 fprintf(stderr,
2977                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2978                                                         port->socket_id);
2979                                                 return -1;
2980                                         }
2981                                         diag = rx_queue_setup(pi, qi,
2982                                              port->nb_rx_desc[qi],
2983                                              port->socket_id,
2984                                              &(port->rxq[qi].conf),
2985                                              mp);
2986                                 }
2987                                 if (diag == 0)
2988                                         continue;
2989
2990                                 /* Fail to setup rx queue, return */
2991                                 if (port->port_status == RTE_PORT_HANDLING)
2992                                         port->port_status = RTE_PORT_STOPPED;
2993                                 else
2994                                         fprintf(stderr,
2995                                                 "Port %d can not be set back to stopped\n",
2996                                                 pi);
2997                                 fprintf(stderr,
2998                                         "Fail to configure port %d rx queues\n",
2999                                         pi);
3000                                 /* try to reconfigure queues next time */
3001                                 port->need_reconfig_queues = 1;
3002                                 return -1;
3003                         }
3004                         /* setup hairpin queues */
3005                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
3006                                 return -1;
3007                 }
3008                 configure_rxtx_dump_callbacks(verbose_level);
3009                 if (clear_ptypes) {
3010                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
3011                                         NULL, 0);
3012                         if (diag < 0)
3013                                 fprintf(stderr,
3014                                         "Port %d: Failed to disable Ptype parsing\n",
3015                                         pi);
3016                 }
3017
3018                 p_pi = pi;
3019                 cnt_pi++;
3020
3021                 /* start port */
3022                 diag = eth_dev_start_mp(pi);
3023                 if (diag < 0) {
3024                         fprintf(stderr, "Fail to start port %d: %s\n",
3025                                 pi, rte_strerror(-diag));
3026
3027                         /* Fail to setup rx queue, return */
3028                         if (port->port_status == RTE_PORT_HANDLING)
3029                                 port->port_status = RTE_PORT_STOPPED;
3030                         else
3031                                 fprintf(stderr,
3032                                         "Port %d can not be set back to stopped\n",
3033                                         pi);
3034                         continue;
3035                 }
3036
3037                 if (port->port_status == RTE_PORT_HANDLING)
3038                         port->port_status = RTE_PORT_STARTED;
3039                 else
3040                         fprintf(stderr, "Port %d can not be set into started\n",
3041                                 pi);
3042
3043                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
3044                         printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
3045                                         RTE_ETHER_ADDR_BYTES(&port->eth_addr));
3046
3047                 /* at least one port started, need checking link status */
3048                 need_check_link_status = 1;
3049
3050                 pl[cfg_pi++] = pi;
3051         }
3052
3053         if (need_check_link_status == 1 && !no_link_check)
3054                 check_all_ports_link_status(RTE_PORT_ALL);
3055         else if (need_check_link_status == 0)
3056                 fprintf(stderr, "Please stop the ports first\n");
3057
3058         if (hairpin_mode & 0xf) {
3059                 uint16_t i;
3060                 int j;
3061
3062                 /* bind all started hairpin ports */
3063                 for (i = 0; i < cfg_pi; i++) {
3064                         pi = pl[i];
3065                         /* bind current Tx to all peer Rx */
3066                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3067                                                         RTE_MAX_ETHPORTS, 1);
3068                         if (peer_pi < 0)
3069                                 return peer_pi;
3070                         for (j = 0; j < peer_pi; j++) {
3071                                 if (!port_is_started(peer_pl[j]))
3072                                         continue;
3073                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
3074                                 if (diag < 0) {
3075                                         fprintf(stderr,
3076                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3077                                                 pi, peer_pl[j],
3078                                                 rte_strerror(-diag));
3079                                         return -1;
3080                                 }
3081                         }
3082                         /* bind all peer Tx to current Rx */
3083                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3084                                                         RTE_MAX_ETHPORTS, 0);
3085                         if (peer_pi < 0)
3086                                 return peer_pi;
3087                         for (j = 0; j < peer_pi; j++) {
3088                                 if (!port_is_started(peer_pl[j]))
3089                                         continue;
3090                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
3091                                 if (diag < 0) {
3092                                         fprintf(stderr,
3093                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3094                                                 peer_pl[j], pi,
3095                                                 rte_strerror(-diag));
3096                                         return -1;
3097                                 }
3098                         }
3099                 }
3100         }
3101
3102         fill_xstats_display_info_for_port(pid);
3103
3104         printf("Done\n");
3105         return 0;
3106 }
3107
3108 void
3109 stop_port(portid_t pid)
3110 {
3111         portid_t pi;
3112         struct rte_port *port;
3113         int need_check_link_status = 0;
3114         portid_t peer_pl[RTE_MAX_ETHPORTS];
3115         int peer_pi;
3116
3117         if (port_id_is_invalid(pid, ENABLED_WARN))
3118                 return;
3119
3120         printf("Stopping ports...\n");
3121
3122         RTE_ETH_FOREACH_DEV(pi) {
3123                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3124                         continue;
3125
3126                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3127                         fprintf(stderr,
3128                                 "Please remove port %d from forwarding configuration.\n",
3129                                 pi);
3130                         continue;
3131                 }
3132
3133                 if (port_is_bonding_slave(pi)) {
3134                         fprintf(stderr,
3135                                 "Please remove port %d from bonded device.\n",
3136                                 pi);
3137                         continue;
3138                 }
3139
3140                 port = &ports[pi];
3141                 if (port->port_status == RTE_PORT_STARTED)
3142                         port->port_status = RTE_PORT_HANDLING;
3143                 else
3144                         continue;
3145
3146                 if (hairpin_mode & 0xf) {
3147                         int j;
3148
3149                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3150                         /* unbind all peer Tx from current Rx */
3151                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3152                                                         RTE_MAX_ETHPORTS, 0);
3153                         if (peer_pi < 0)
3154                                 continue;
3155                         for (j = 0; j < peer_pi; j++) {
3156                                 if (!port_is_started(peer_pl[j]))
3157                                         continue;
3158                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
3159                         }
3160                 }
3161
3162                 if (port->flow_list)
3163                         port_flow_flush(pi);
3164
3165                 if (eth_dev_stop_mp(pi) != 0)
3166                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3167                                 pi);
3168
3169                 if (port->port_status == RTE_PORT_HANDLING)
3170                         port->port_status = RTE_PORT_STOPPED;
3171                 else
3172                         fprintf(stderr, "Port %d can not be set into stopped\n",
3173                                 pi);
3174                 need_check_link_status = 1;
3175         }
3176         if (need_check_link_status && !no_link_check)
3177                 check_all_ports_link_status(RTE_PORT_ALL);
3178
3179         printf("Done\n");
3180 }
3181
3182 static void
3183 remove_invalid_ports_in(portid_t *array, portid_t *total)
3184 {
3185         portid_t i;
3186         portid_t new_total = 0;
3187
3188         for (i = 0; i < *total; i++)
3189                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3190                         array[new_total] = array[i];
3191                         new_total++;
3192                 }
3193         *total = new_total;
3194 }
3195
3196 static void
3197 remove_invalid_ports(void)
3198 {
3199         remove_invalid_ports_in(ports_ids, &nb_ports);
3200         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3201         nb_cfg_ports = nb_fwd_ports;
3202 }
3203
3204 void
3205 close_port(portid_t pid)
3206 {
3207         portid_t pi;
3208         struct rte_port *port;
3209
3210         if (port_id_is_invalid(pid, ENABLED_WARN))
3211                 return;
3212
3213         printf("Closing ports...\n");
3214
3215         RTE_ETH_FOREACH_DEV(pi) {
3216                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3217                         continue;
3218
3219                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3220                         fprintf(stderr,
3221                                 "Please remove port %d from forwarding configuration.\n",
3222                                 pi);
3223                         continue;
3224                 }
3225
3226                 if (port_is_bonding_slave(pi)) {
3227                         fprintf(stderr,
3228                                 "Please remove port %d from bonded device.\n",
3229                                 pi);
3230                         continue;
3231                 }
3232
3233                 port = &ports[pi];
3234                 if (port->port_status == RTE_PORT_CLOSED) {
3235                         fprintf(stderr, "Port %d is already closed\n", pi);
3236                         continue;
3237                 }
3238
3239                 if (is_proc_primary()) {
3240                         port_flow_flush(pi);
3241                         port_flex_item_flush(pi);
3242                         port_action_handle_flush(pi);
3243                         rte_eth_dev_close(pi);
3244                 }
3245
3246                 free_xstats_display_info(pi);
3247         }
3248
3249         remove_invalid_ports();
3250         printf("Done\n");
3251 }
3252
3253 void
3254 reset_port(portid_t pid)
3255 {
3256         int diag;
3257         portid_t pi;
3258         struct rte_port *port;
3259
3260         if (port_id_is_invalid(pid, ENABLED_WARN))
3261                 return;
3262
3263         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3264                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3265                 fprintf(stderr,
3266                         "Can not reset port(s), please stop port(s) first.\n");
3267                 return;
3268         }
3269
3270         printf("Resetting ports...\n");
3271
3272         RTE_ETH_FOREACH_DEV(pi) {
3273                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3274                         continue;
3275
3276                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3277                         fprintf(stderr,
3278                                 "Please remove port %d from forwarding configuration.\n",
3279                                 pi);
3280                         continue;
3281                 }
3282
3283                 if (port_is_bonding_slave(pi)) {
3284                         fprintf(stderr,
3285                                 "Please remove port %d from bonded device.\n",
3286                                 pi);
3287                         continue;
3288                 }
3289
3290                 diag = rte_eth_dev_reset(pi);
3291                 if (diag == 0) {
3292                         port = &ports[pi];
3293                         port->need_reconfig = 1;
3294                         port->need_reconfig_queues = 1;
3295                 } else {
3296                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3297                                 pi, diag);
3298                 }
3299         }
3300
3301         printf("Done\n");
3302 }
3303
3304 void
3305 attach_port(char *identifier)
3306 {
3307         portid_t pi;
3308         struct rte_dev_iterator iterator;
3309
3310         printf("Attaching a new port...\n");
3311
3312         if (identifier == NULL) {
3313                 fprintf(stderr, "Invalid parameters are specified\n");
3314                 return;
3315         }
3316
3317         if (rte_dev_probe(identifier) < 0) {
3318                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3319                 return;
3320         }
3321
3322         /* first attach mode: event */
3323         if (setup_on_probe_event) {
3324                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3325                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3326                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
3327                                         ports[pi].need_setup != 0)
3328                                 setup_attached_port(pi);
3329                 return;
3330         }
3331
3332         /* second attach mode: iterator */
3333         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3334                 /* setup ports matching the devargs used for probing */
3335                 if (port_is_forwarding(pi))
3336                         continue; /* port was already attached before */
3337                 setup_attached_port(pi);
3338         }
3339 }
3340
3341 static void
3342 setup_attached_port(portid_t pi)
3343 {
3344         unsigned int socket_id;
3345         int ret;
3346
3347         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3348         /* if socket_id is invalid, set to the first available socket. */
3349         if (check_socket_id(socket_id) < 0)
3350                 socket_id = socket_ids[0];
3351         reconfig(pi, socket_id);
3352         ret = rte_eth_promiscuous_enable(pi);
3353         if (ret != 0)
3354                 fprintf(stderr,
3355                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3356                         pi, rte_strerror(-ret));
3357
3358         ports_ids[nb_ports++] = pi;
3359         fwd_ports_ids[nb_fwd_ports++] = pi;
3360         nb_cfg_ports = nb_fwd_ports;
3361         ports[pi].need_setup = 0;
3362         ports[pi].port_status = RTE_PORT_STOPPED;
3363
3364         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3365         printf("Done\n");
3366 }
3367
3368 static void
3369 detach_device(struct rte_device *dev)
3370 {
3371         portid_t sibling;
3372
3373         if (dev == NULL) {
3374                 fprintf(stderr, "Device already removed\n");
3375                 return;
3376         }
3377
3378         printf("Removing a device...\n");
3379
3380         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3381                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3382                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3383                                 fprintf(stderr, "Port %u not stopped\n",
3384                                         sibling);
3385                                 return;
3386                         }
3387                         port_flow_flush(sibling);
3388                 }
3389         }
3390
3391         if (rte_dev_remove(dev) < 0) {
3392                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3393                 return;
3394         }
3395         remove_invalid_ports();
3396
3397         printf("Device is detached\n");
3398         printf("Now total ports is %d\n", nb_ports);
3399         printf("Done\n");
3400         return;
3401 }
3402
3403 void
3404 detach_port_device(portid_t port_id)
3405 {
3406         int ret;
3407         struct rte_eth_dev_info dev_info;
3408
3409         if (port_id_is_invalid(port_id, ENABLED_WARN))
3410                 return;
3411
3412         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3413                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3414                         fprintf(stderr, "Port not stopped\n");
3415                         return;
3416                 }
3417                 fprintf(stderr, "Port was not closed\n");
3418         }
3419
3420         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3421         if (ret != 0) {
3422                 TESTPMD_LOG(ERR,
3423                         "Failed to get device info for port %d, not detaching\n",
3424                         port_id);
3425                 return;
3426         }
3427         detach_device(dev_info.device);
3428 }
3429
3430 void
3431 detach_devargs(char *identifier)
3432 {
3433         struct rte_dev_iterator iterator;
3434         struct rte_devargs da;
3435         portid_t port_id;
3436
3437         printf("Removing a device...\n");
3438
3439         memset(&da, 0, sizeof(da));
3440         if (rte_devargs_parsef(&da, "%s", identifier)) {
3441                 fprintf(stderr, "cannot parse identifier\n");
3442                 return;
3443         }
3444
3445         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3446                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3447                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3448                                 fprintf(stderr, "Port %u not stopped\n",
3449                                         port_id);
3450                                 rte_eth_iterator_cleanup(&iterator);
3451                                 rte_devargs_reset(&da);
3452                                 return;
3453                         }
3454                         port_flow_flush(port_id);
3455                 }
3456         }
3457
3458         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3459                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3460                             da.name, da.bus->name);
3461                 rte_devargs_reset(&da);
3462                 return;
3463         }
3464
3465         remove_invalid_ports();
3466
3467         printf("Device %s is detached\n", identifier);
3468         printf("Now total ports is %d\n", nb_ports);
3469         printf("Done\n");
3470         rte_devargs_reset(&da);
3471 }
3472
3473 void
3474 pmd_test_exit(void)
3475 {
3476         portid_t pt_id;
3477         unsigned int i;
3478         int ret;
3479
3480         if (test_done == 0)
3481                 stop_packet_forwarding();
3482
3483 #ifndef RTE_EXEC_ENV_WINDOWS
3484         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3485                 if (mempools[i]) {
3486                         if (mp_alloc_type == MP_ALLOC_ANON)
3487                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3488                                                      NULL);
3489                 }
3490         }
3491 #endif
3492         if (ports != NULL) {
3493                 no_link_check = 1;
3494                 RTE_ETH_FOREACH_DEV(pt_id) {
3495                         printf("\nStopping port %d...\n", pt_id);
3496                         fflush(stdout);
3497                         stop_port(pt_id);
3498                 }
3499                 RTE_ETH_FOREACH_DEV(pt_id) {
3500                         printf("\nShutting down port %d...\n", pt_id);
3501                         fflush(stdout);
3502                         close_port(pt_id);
3503                 }
3504         }
3505
3506         if (hot_plug) {
3507                 ret = rte_dev_event_monitor_stop();
3508                 if (ret) {
3509                         RTE_LOG(ERR, EAL,
3510                                 "fail to stop device event monitor.");
3511                         return;
3512                 }
3513
3514                 ret = rte_dev_event_callback_unregister(NULL,
3515                         dev_event_callback, NULL);
3516                 if (ret < 0) {
3517                         RTE_LOG(ERR, EAL,
3518                                 "fail to unregister device event callback.\n");
3519                         return;
3520                 }
3521
3522                 ret = rte_dev_hotplug_handle_disable();
3523                 if (ret) {
3524                         RTE_LOG(ERR, EAL,
3525                                 "fail to disable hotplug handling.\n");
3526                         return;
3527                 }
3528         }
3529         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3530                 if (mempools[i])
3531                         mempool_free_mp(mempools[i]);
3532         }
3533         free(xstats_display);
3534
3535         printf("\nBye...\n");
3536 }
3537
3538 typedef void (*cmd_func_t)(void);
3539 struct pmd_test_command {
3540         const char *cmd_name;
3541         cmd_func_t cmd_func;
3542 };
3543
3544 /* Check the link status of all ports in up to 9s, and print them finally */
3545 static void
3546 check_all_ports_link_status(uint32_t port_mask)
3547 {
3548 #define CHECK_INTERVAL 100 /* 100ms */
3549 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3550         portid_t portid;
3551         uint8_t count, all_ports_up, print_flag = 0;
3552         struct rte_eth_link link;
3553         int ret;
3554         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3555
3556         printf("Checking link statuses...\n");
3557         fflush(stdout);
3558         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3559                 all_ports_up = 1;
3560                 RTE_ETH_FOREACH_DEV(portid) {
3561                         if ((port_mask & (1 << portid)) == 0)
3562                                 continue;
3563                         memset(&link, 0, sizeof(link));
3564                         ret = rte_eth_link_get_nowait(portid, &link);
3565                         if (ret < 0) {
3566                                 all_ports_up = 0;
3567                                 if (print_flag == 1)
3568                                         fprintf(stderr,
3569                                                 "Port %u link get failed: %s\n",
3570                                                 portid, rte_strerror(-ret));
3571                                 continue;
3572                         }
3573                         /* print link status if flag set */
3574                         if (print_flag == 1) {
3575                                 rte_eth_link_to_str(link_status,
3576                                         sizeof(link_status), &link);
3577                                 printf("Port %d %s\n", portid, link_status);
3578                                 continue;
3579                         }
3580                         /* clear all_ports_up flag if any link down */
3581                         if (link.link_status == RTE_ETH_LINK_DOWN) {
3582                                 all_ports_up = 0;
3583                                 break;
3584                         }
3585                 }
3586                 /* after finally printing all link status, get out */
3587                 if (print_flag == 1)
3588                         break;
3589
3590                 if (all_ports_up == 0) {
3591                         fflush(stdout);
3592                         rte_delay_ms(CHECK_INTERVAL);
3593                 }
3594
3595                 /* set the print_flag if all ports up or timeout */
3596                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3597                         print_flag = 1;
3598                 }
3599
3600                 if (lsc_interrupt)
3601                         break;
3602         }
3603 }
3604
3605 static void
3606 rmv_port_callback(void *arg)
3607 {
3608         int need_to_start = 0;
3609         int org_no_link_check = no_link_check;
3610         portid_t port_id = (intptr_t)arg;
3611         struct rte_eth_dev_info dev_info;
3612         int ret;
3613
3614         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3615
3616         if (!test_done && port_is_forwarding(port_id)) {
3617                 need_to_start = 1;
3618                 stop_packet_forwarding();
3619         }
3620         no_link_check = 1;
3621         stop_port(port_id);
3622         no_link_check = org_no_link_check;
3623
3624         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3625         if (ret != 0)
3626                 TESTPMD_LOG(ERR,
3627                         "Failed to get device info for port %d, not detaching\n",
3628                         port_id);
3629         else {
3630                 struct rte_device *device = dev_info.device;
3631                 close_port(port_id);
3632                 detach_device(device); /* might be already removed or have more ports */
3633         }
3634         if (need_to_start)
3635                 start_packet_forwarding(0);
3636 }
3637
3638 /* This function is used by the interrupt thread */
3639 static int
3640 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3641                   void *ret_param)
3642 {
3643         RTE_SET_USED(param);
3644         RTE_SET_USED(ret_param);
3645
3646         if (type >= RTE_ETH_EVENT_MAX) {
3647                 fprintf(stderr,
3648                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3649                         port_id, __func__, type);
3650                 fflush(stderr);
3651         } else if (event_print_mask & (UINT32_C(1) << type)) {
3652                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3653                         eth_event_desc[type]);
3654                 fflush(stdout);
3655         }
3656
3657         switch (type) {
3658         case RTE_ETH_EVENT_NEW:
3659                 ports[port_id].need_setup = 1;
3660                 ports[port_id].port_status = RTE_PORT_HANDLING;
3661                 break;
3662         case RTE_ETH_EVENT_INTR_RMV:
3663                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3664                         break;
3665                 if (rte_eal_alarm_set(100000,
3666                                 rmv_port_callback, (void *)(intptr_t)port_id))
3667                         fprintf(stderr,
3668                                 "Could not set up deferred device removal\n");
3669                 break;
3670         case RTE_ETH_EVENT_DESTROY:
3671                 ports[port_id].port_status = RTE_PORT_CLOSED;
3672                 printf("Port %u is closed\n", port_id);
3673                 break;
3674         default:
3675                 break;
3676         }
3677         return 0;
3678 }
3679
3680 static int
3681 register_eth_event_callback(void)
3682 {
3683         int ret;
3684         enum rte_eth_event_type event;
3685
3686         for (event = RTE_ETH_EVENT_UNKNOWN;
3687                         event < RTE_ETH_EVENT_MAX; event++) {
3688                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3689                                 event,
3690                                 eth_event_callback,
3691                                 NULL);
3692                 if (ret != 0) {
3693                         TESTPMD_LOG(ERR, "Failed to register callback for "
3694                                         "%s event\n", eth_event_desc[event]);
3695                         return -1;
3696                 }
3697         }
3698
3699         return 0;
3700 }
3701
3702 /* This function is used by the interrupt thread */
3703 static void
3704 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3705                              __rte_unused void *arg)
3706 {
3707         uint16_t port_id;
3708         int ret;
3709
3710         if (type >= RTE_DEV_EVENT_MAX) {
3711                 fprintf(stderr, "%s called upon invalid event %d\n",
3712                         __func__, type);
3713                 fflush(stderr);
3714         }
3715
3716         switch (type) {
3717         case RTE_DEV_EVENT_REMOVE:
3718                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3719                         device_name);
3720                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3721                 if (ret) {
3722                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3723                                 device_name);
3724                         return;
3725                 }
3726                 /*
3727                  * Because the user's callback is invoked in eal interrupt
3728                  * callback, the interrupt callback need to be finished before
3729                  * it can be unregistered when detaching device. So finish
3730                  * callback soon and use a deferred removal to detach device
3731                  * is need. It is a workaround, once the device detaching be
3732                  * moved into the eal in the future, the deferred removal could
3733                  * be deleted.
3734                  */
3735                 if (rte_eal_alarm_set(100000,
3736                                 rmv_port_callback, (void *)(intptr_t)port_id))
3737                         RTE_LOG(ERR, EAL,
3738                                 "Could not set up deferred device removal\n");
3739                 break;
3740         case RTE_DEV_EVENT_ADD:
3741                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3742                         device_name);
3743                 /* TODO: After finish kernel driver binding,
3744                  * begin to attach port.
3745                  */
3746                 break;
3747         default:
3748                 break;
3749         }
3750 }
3751
3752 static void
3753 rxtx_port_config(portid_t pid)
3754 {
3755         uint16_t qid;
3756         uint64_t offloads;
3757         struct rte_port *port = &ports[pid];
3758
3759         for (qid = 0; qid < nb_rxq; qid++) {
3760                 offloads = port->rxq[qid].conf.offloads;
3761                 port->rxq[qid].conf = port->dev_info.default_rxconf;
3762
3763                 if (rxq_share > 0 &&
3764                     (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3765                         /* Non-zero share group to enable RxQ share. */
3766                         port->rxq[qid].conf.share_group = pid / rxq_share + 1;
3767                         port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */
3768                 }
3769
3770                 if (offloads != 0)
3771                         port->rxq[qid].conf.offloads = offloads;
3772
3773                 /* Check if any Rx parameters have been passed */
3774                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3775                         port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh;
3776
3777                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3778                         port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh;
3779
3780                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3781                         port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh;
3782
3783                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3784                         port->rxq[qid].conf.rx_free_thresh = rx_free_thresh;
3785
3786                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3787                         port->rxq[qid].conf.rx_drop_en = rx_drop_en;
3788
3789                 port->nb_rx_desc[qid] = nb_rxd;
3790         }
3791
3792         for (qid = 0; qid < nb_txq; qid++) {
3793                 offloads = port->txq[qid].conf.offloads;
3794                 port->txq[qid].conf = port->dev_info.default_txconf;
3795                 if (offloads != 0)
3796                         port->txq[qid].conf.offloads = offloads;
3797
3798                 /* Check if any Tx parameters have been passed */
3799                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3800                         port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh;
3801
3802                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3803                         port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh;
3804
3805                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3806                         port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh;
3807
3808                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3809                         port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh;
3810
3811                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3812                         port->txq[qid].conf.tx_free_thresh = tx_free_thresh;
3813
3814                 port->nb_tx_desc[qid] = nb_txd;
3815         }
3816 }
3817
3818 /*
3819  * Helper function to set MTU from frame size
3820  *
3821  * port->dev_info should be set before calling this function.
3822  *
3823  * return 0 on success, negative on error
3824  */
3825 int
3826 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3827 {
3828         struct rte_port *port = &ports[portid];
3829         uint32_t eth_overhead;
3830         uint16_t mtu, new_mtu;
3831
3832         eth_overhead = get_eth_overhead(&port->dev_info);
3833
3834         if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3835                 printf("Failed to get MTU for port %u\n", portid);
3836                 return -1;
3837         }
3838
3839         new_mtu = max_rx_pktlen - eth_overhead;
3840
3841         if (mtu == new_mtu)
3842                 return 0;
3843
3844         if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3845                 fprintf(stderr,
3846                         "Failed to set MTU to %u for port %u\n",
3847                         new_mtu, portid);
3848                 return -1;
3849         }
3850
3851         port->dev_conf.rxmode.mtu = new_mtu;
3852
3853         return 0;
3854 }
3855
3856 void
3857 init_port_config(void)
3858 {
3859         portid_t pid;
3860         struct rte_port *port;
3861         int ret, i;
3862
3863         RTE_ETH_FOREACH_DEV(pid) {
3864                 port = &ports[pid];
3865                 port->dev_conf.fdir_conf = fdir_conf;
3866
3867                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3868                 if (ret != 0)
3869                         return;
3870
3871                 if (nb_rxq > 1) {
3872                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3873                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3874                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3875                 } else {
3876                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3877                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3878                 }
3879
3880                 if (port->dcb_flag == 0) {
3881                         if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3882                                 port->dev_conf.rxmode.mq_mode =
3883                                         (enum rte_eth_rx_mq_mode)
3884                                                 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3885                         } else {
3886                                 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3887                                 port->dev_conf.rxmode.offloads &=
3888                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3889
3890                                 for (i = 0;
3891                                      i < port->dev_info.nb_rx_queues;
3892                                      i++)
3893                                         port->rxq[i].conf.offloads &=
3894                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3895                         }
3896                 }
3897
3898                 rxtx_port_config(pid);
3899
3900                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3901                 if (ret != 0)
3902                         return;
3903
3904 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3905                 rte_pmd_ixgbe_bypass_init(pid);
3906 #endif
3907
3908                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3909                         port->dev_conf.intr_conf.lsc = 1;
3910                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3911                         port->dev_conf.intr_conf.rmv = 1;
3912         }
3913 }
3914
3915 void set_port_slave_flag(portid_t slave_pid)
3916 {
3917         struct rte_port *port;
3918
3919         port = &ports[slave_pid];
3920         port->slave_flag = 1;
3921 }
3922
3923 void clear_port_slave_flag(portid_t slave_pid)
3924 {
3925         struct rte_port *port;
3926
3927         port = &ports[slave_pid];
3928         port->slave_flag = 0;
3929 }
3930
3931 uint8_t port_is_bonding_slave(portid_t slave_pid)
3932 {
3933         struct rte_port *port;
3934         struct rte_eth_dev_info dev_info;
3935         int ret;
3936
3937         port = &ports[slave_pid];
3938         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3939         if (ret != 0) {
3940                 TESTPMD_LOG(ERR,
3941                         "Failed to get device info for port id %d,"
3942                         "cannot determine if the port is a bonded slave",
3943                         slave_pid);
3944                 return 0;
3945         }
3946         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3947                 return 1;
3948         return 0;
3949 }
3950
3951 const uint16_t vlan_tags[] = {
3952                 0,  1,  2,  3,  4,  5,  6,  7,
3953                 8,  9, 10, 11,  12, 13, 14, 15,
3954                 16, 17, 18, 19, 20, 21, 22, 23,
3955                 24, 25, 26, 27, 28, 29, 30, 31
3956 };
3957
3958 static  int
3959 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3960                  enum dcb_mode_enable dcb_mode,
3961                  enum rte_eth_nb_tcs num_tcs,
3962                  uint8_t pfc_en)
3963 {
3964         uint8_t i;
3965         int32_t rc;
3966         struct rte_eth_rss_conf rss_conf;
3967
3968         /*
3969          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3970          * given above, and the number of traffic classes available for use.
3971          */
3972         if (dcb_mode == DCB_VT_ENABLED) {
3973                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3974                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3975                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3976                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3977
3978                 /* VMDQ+DCB RX and TX configurations */
3979                 vmdq_rx_conf->enable_default_pool = 0;
3980                 vmdq_rx_conf->default_pool = 0;
3981                 vmdq_rx_conf->nb_queue_pools =
3982                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3983                 vmdq_tx_conf->nb_queue_pools =
3984                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3985
3986                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3987                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3988                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3989                         vmdq_rx_conf->pool_map[i].pools =
3990                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3991                 }
3992                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3993                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3994                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3995                 }
3996
3997                 /* set DCB mode of RX and TX of multiple queues */
3998                 eth_conf->rxmode.mq_mode =
3999                                 (enum rte_eth_rx_mq_mode)
4000                                         (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
4001                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
4002         } else {
4003                 struct rte_eth_dcb_rx_conf *rx_conf =
4004                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
4005                 struct rte_eth_dcb_tx_conf *tx_conf =
4006                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
4007
4008                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
4009
4010                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
4011                 if (rc != 0)
4012                         return rc;
4013
4014                 rx_conf->nb_tcs = num_tcs;
4015                 tx_conf->nb_tcs = num_tcs;
4016
4017                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
4018                         rx_conf->dcb_tc[i] = i % num_tcs;
4019                         tx_conf->dcb_tc[i] = i % num_tcs;
4020                 }
4021
4022                 eth_conf->rxmode.mq_mode =
4023                                 (enum rte_eth_rx_mq_mode)
4024                                         (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
4025                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
4026                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
4027         }
4028
4029         if (pfc_en)
4030                 eth_conf->dcb_capability_en =
4031                                 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
4032         else
4033                 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
4034
4035         return 0;
4036 }
4037
4038 int
4039 init_port_dcb_config(portid_t pid,
4040                      enum dcb_mode_enable dcb_mode,
4041                      enum rte_eth_nb_tcs num_tcs,
4042                      uint8_t pfc_en)
4043 {
4044         struct rte_eth_conf port_conf;
4045         struct rte_port *rte_port;
4046         int retval;
4047         uint16_t i;
4048
4049         if (num_procs > 1) {
4050                 printf("The multi-process feature doesn't support dcb.\n");
4051                 return -ENOTSUP;
4052         }
4053         rte_port = &ports[pid];
4054
4055         /* retain the original device configuration. */
4056         memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
4057
4058         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
4059         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
4060         if (retval < 0)
4061                 return retval;
4062         port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4063         /* remove RSS HASH offload for DCB in vt mode */
4064         if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
4065                 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4066                 for (i = 0; i < nb_rxq; i++)
4067                         rte_port->rxq[i].conf.offloads &=
4068                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4069         }
4070
4071         /* re-configure the device . */
4072         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
4073         if (retval < 0)
4074                 return retval;
4075
4076         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
4077         if (retval != 0)
4078                 return retval;
4079
4080         /* If dev_info.vmdq_pool_base is greater than 0,
4081          * the queue id of vmdq pools is started after pf queues.
4082          */
4083         if (dcb_mode == DCB_VT_ENABLED &&
4084             rte_port->dev_info.vmdq_pool_base > 0) {
4085                 fprintf(stderr,
4086                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
4087                         pid);
4088                 return -1;
4089         }
4090
4091         /* Assume the ports in testpmd have the same dcb capability
4092          * and has the same number of rxq and txq in dcb mode
4093          */
4094         if (dcb_mode == DCB_VT_ENABLED) {
4095                 if (rte_port->dev_info.max_vfs > 0) {
4096                         nb_rxq = rte_port->dev_info.nb_rx_queues;
4097                         nb_txq = rte_port->dev_info.nb_tx_queues;
4098                 } else {
4099                         nb_rxq = rte_port->dev_info.max_rx_queues;
4100                         nb_txq = rte_port->dev_info.max_tx_queues;
4101                 }
4102         } else {
4103                 /*if vt is disabled, use all pf queues */
4104                 if (rte_port->dev_info.vmdq_pool_base == 0) {
4105                         nb_rxq = rte_port->dev_info.max_rx_queues;
4106                         nb_txq = rte_port->dev_info.max_tx_queues;
4107                 } else {
4108                         nb_rxq = (queueid_t)num_tcs;
4109                         nb_txq = (queueid_t)num_tcs;
4110
4111                 }
4112         }
4113         rx_free_thresh = 64;
4114
4115         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4116
4117         rxtx_port_config(pid);
4118         /* VLAN filter */
4119         rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4120         for (i = 0; i < RTE_DIM(vlan_tags); i++)
4121                 rx_vft_set(pid, vlan_tags[i], 1);
4122
4123         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4124         if (retval != 0)
4125                 return retval;
4126
4127         rte_port->dcb_flag = 1;
4128
4129         /* Enter DCB configuration status */
4130         dcb_config = 1;
4131
4132         return 0;
4133 }
4134
4135 static void
4136 init_port(void)
4137 {
4138         int i;
4139
4140         /* Configuration of Ethernet ports. */
4141         ports = rte_zmalloc("testpmd: ports",
4142                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4143                             RTE_CACHE_LINE_SIZE);
4144         if (ports == NULL) {
4145                 rte_exit(EXIT_FAILURE,
4146                                 "rte_zmalloc(%d struct rte_port) failed\n",
4147                                 RTE_MAX_ETHPORTS);
4148         }
4149         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4150                 ports[i].xstats_info.allocated = false;
4151         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4152                 LIST_INIT(&ports[i].flow_tunnel_list);
4153         /* Initialize ports NUMA structures */
4154         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4155         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4156         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4157 }
4158
4159 static void
4160 force_quit(void)
4161 {
4162         pmd_test_exit();
4163         prompt_exit();
4164 }
4165
4166 static void
4167 print_stats(void)
4168 {
4169         uint8_t i;
4170         const char clr[] = { 27, '[', '2', 'J', '\0' };
4171         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4172
4173         /* Clear screen and move to top left */
4174         printf("%s%s", clr, top_left);
4175
4176         printf("\nPort statistics ====================================");
4177         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4178                 nic_stats_display(fwd_ports_ids[i]);
4179
4180         fflush(stdout);
4181 }
4182
4183 static void
4184 signal_handler(int signum)
4185 {
4186         if (signum == SIGINT || signum == SIGTERM) {
4187                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4188                         signum);
4189 #ifdef RTE_LIB_PDUMP
4190                 /* uninitialize packet capture framework */
4191                 rte_pdump_uninit();
4192 #endif
4193 #ifdef RTE_LIB_LATENCYSTATS
4194                 if (latencystats_enabled != 0)
4195                         rte_latencystats_uninit();
4196 #endif
4197                 force_quit();
4198                 /* Set flag to indicate the force termination. */
4199                 f_quit = 1;
4200                 /* exit with the expected status */
4201 #ifndef RTE_EXEC_ENV_WINDOWS
4202                 signal(signum, SIG_DFL);
4203                 kill(getpid(), signum);
4204 #endif
4205         }
4206 }
4207
4208 int
4209 main(int argc, char** argv)
4210 {
4211         int diag;
4212         portid_t port_id;
4213         uint16_t count;
4214         int ret;
4215
4216         signal(SIGINT, signal_handler);
4217         signal(SIGTERM, signal_handler);
4218
4219         testpmd_logtype = rte_log_register("testpmd");
4220         if (testpmd_logtype < 0)
4221                 rte_exit(EXIT_FAILURE, "Cannot register log type");
4222         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4223
4224         diag = rte_eal_init(argc, argv);
4225         if (diag < 0)
4226                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4227                          rte_strerror(rte_errno));
4228
4229         ret = register_eth_event_callback();
4230         if (ret != 0)
4231                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4232
4233 #ifdef RTE_LIB_PDUMP
4234         /* initialize packet capture framework */
4235         rte_pdump_init();
4236 #endif
4237
4238         count = 0;
4239         RTE_ETH_FOREACH_DEV(port_id) {
4240                 ports_ids[count] = port_id;
4241                 count++;
4242         }
4243         nb_ports = (portid_t) count;
4244         if (nb_ports == 0)
4245                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4246
4247         /* allocate port structures, and init them */
4248         init_port();
4249
4250         set_def_fwd_config();
4251         if (nb_lcores == 0)
4252                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4253                          "Check the core mask argument\n");
4254
4255         /* Bitrate/latency stats disabled by default */
4256 #ifdef RTE_LIB_BITRATESTATS
4257         bitrate_enabled = 0;
4258 #endif
4259 #ifdef RTE_LIB_LATENCYSTATS
4260         latencystats_enabled = 0;
4261 #endif
4262
4263         /* on FreeBSD, mlockall() is disabled by default */
4264 #ifdef RTE_EXEC_ENV_FREEBSD
4265         do_mlockall = 0;
4266 #else
4267         do_mlockall = 1;
4268 #endif
4269
4270         argc -= diag;
4271         argv += diag;
4272         if (argc > 1)
4273                 launch_args_parse(argc, argv);
4274
4275 #ifndef RTE_EXEC_ENV_WINDOWS
4276         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4277                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4278                         strerror(errno));
4279         }
4280 #endif
4281
4282         if (tx_first && interactive)
4283                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4284                                 "interactive mode.\n");
4285
4286         if (tx_first && lsc_interrupt) {
4287                 fprintf(stderr,
4288                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4289                 lsc_interrupt = 0;
4290         }
4291
4292         if (!nb_rxq && !nb_txq)
4293                 fprintf(stderr,
4294                         "Warning: Either rx or tx queues should be non-zero\n");
4295
4296         if (nb_rxq > 1 && nb_rxq > nb_txq)
4297                 fprintf(stderr,
4298                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4299                         nb_rxq, nb_txq);
4300
4301         init_config();
4302
4303         if (hot_plug) {
4304                 ret = rte_dev_hotplug_handle_enable();
4305                 if (ret) {
4306                         RTE_LOG(ERR, EAL,
4307                                 "fail to enable hotplug handling.");
4308                         return -1;
4309                 }
4310
4311                 ret = rte_dev_event_monitor_start();
4312                 if (ret) {
4313                         RTE_LOG(ERR, EAL,
4314                                 "fail to start device event monitoring.");
4315                         return -1;
4316                 }
4317
4318                 ret = rte_dev_event_callback_register(NULL,
4319                         dev_event_callback, NULL);
4320                 if (ret) {
4321                         RTE_LOG(ERR, EAL,
4322                                 "fail  to register device event callback\n");
4323                         return -1;
4324                 }
4325         }
4326
4327         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4328                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4329
4330         /* set all ports to promiscuous mode by default */
4331         RTE_ETH_FOREACH_DEV(port_id) {
4332                 ret = rte_eth_promiscuous_enable(port_id);
4333                 if (ret != 0)
4334                         fprintf(stderr,
4335                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4336                                 port_id, rte_strerror(-ret));
4337         }
4338
4339 #ifdef RTE_LIB_METRICS
4340         /* Init metrics library */
4341         rte_metrics_init(rte_socket_id());
4342 #endif
4343
4344 #ifdef RTE_LIB_LATENCYSTATS
4345         if (latencystats_enabled != 0) {
4346                 int ret = rte_latencystats_init(1, NULL);
4347                 if (ret)
4348                         fprintf(stderr,
4349                                 "Warning: latencystats init() returned error %d\n",
4350                                 ret);
4351                 fprintf(stderr, "Latencystats running on lcore %d\n",
4352                         latencystats_lcore_id);
4353         }
4354 #endif
4355
4356         /* Setup bitrate stats */
4357 #ifdef RTE_LIB_BITRATESTATS
4358         if (bitrate_enabled != 0) {
4359                 bitrate_data = rte_stats_bitrate_create();
4360                 if (bitrate_data == NULL)
4361                         rte_exit(EXIT_FAILURE,
4362                                 "Could not allocate bitrate data.\n");
4363                 rte_stats_bitrate_reg(bitrate_data);
4364         }
4365 #endif
4366 #ifdef RTE_LIB_CMDLINE
4367         if (init_cmdline() != 0)
4368                 rte_exit(EXIT_FAILURE,
4369                         "Could not initialise cmdline context.\n");
4370
4371         if (strlen(cmdline_filename) != 0)
4372                 cmdline_read_from_file(cmdline_filename);
4373
4374         if (interactive == 1) {
4375                 if (auto_start) {
4376                         printf("Start automatic packet forwarding\n");
4377                         start_packet_forwarding(0);
4378                 }
4379                 prompt();
4380                 pmd_test_exit();
4381         } else
4382 #endif
4383         {
4384                 char c;
4385                 int rc;
4386
4387                 f_quit = 0;
4388
4389                 printf("No commandline core given, start packet forwarding\n");
4390                 start_packet_forwarding(tx_first);
4391                 if (stats_period != 0) {
4392                         uint64_t prev_time = 0, cur_time, diff_time = 0;
4393                         uint64_t timer_period;
4394
4395                         /* Convert to number of cycles */
4396                         timer_period = stats_period * rte_get_timer_hz();
4397
4398                         while (f_quit == 0) {
4399                                 cur_time = rte_get_timer_cycles();
4400                                 diff_time += cur_time - prev_time;
4401
4402                                 if (diff_time >= timer_period) {
4403                                         print_stats();
4404                                         /* Reset the timer */
4405                                         diff_time = 0;
4406                                 }
4407                                 /* Sleep to avoid unnecessary checks */
4408                                 prev_time = cur_time;
4409                                 rte_delay_us_sleep(US_PER_S);
4410                         }
4411                 }
4412
4413                 printf("Press enter to exit\n");
4414                 rc = read(0, &c, 1);
4415                 pmd_test_exit();
4416                 if (rc < 0)
4417                         return 1;
4418         }
4419
4420         ret = rte_eal_cleanup();
4421         if (ret != 0)
4422                 rte_exit(EXIT_FAILURE,
4423                          "EAL cleanup failed: %s\n", strerror(-ret));
4424
4425         return EXIT_SUCCESS;
4426 }