test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_NET_IXGBE
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIB_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #ifdef RTE_LIB_METRICS
  58 #include <rte_metrics.h>
  59 #endif
  60 #ifdef RTE_LIB_BITRATESTATS
  61 #include <rte_bitrate.h>
  62 #endif
  63 #ifdef RTE_LIB_LATENCYSTATS
  64 #include <rte_latencystats.h>
  65 #endif
  66 #ifdef RTE_EXEC_ENV_WINDOWS
  67 #include <process.h>
  68 #endif
  69 #ifdef RTE_NET_BOND
  70 #include <rte_eth_bond.h>
  71 #endif
  72
  73 #include "testpmd.h"
  74
  75 #ifndef MAP_HUGETLB
  76 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  77 #define HUGE_FLAG (0x40000)
  78 #else
  79 #define HUGE_FLAG MAP_HUGETLB
  80 #endif
  81
  82 #ifndef MAP_HUGE_SHIFT
  83 /* older kernels (or FreeBSD) will not have this define */
  84 #define HUGE_SHIFT (26)
  85 #else
  86 #define HUGE_SHIFT MAP_HUGE_SHIFT
  87 #endif
  88
  89 #define EXTMEM_HEAP_NAME "extmem"
  90 /*
  91  * Zone size with the malloc overhead (max of debug and release variants)
  92  * must fit into the smallest supported hugepage size (2M),
  93  * so that an IOVA-contiguous zone of this size can always be allocated
  94  * if there are free 2M hugepages.
  95  */
  96 #define EXTBUF_ZONE_SIZE (RTE_PGSIZE_2M - 4 * RTE_CACHE_LINE_SIZE)
  97
  98 uint16_t verbose_level = 0; /**< Silent by default. */
  99 int testpmd_logtype; /**< Log type for testpmd logs */
 100
 101 /* use main core for command line ? */
 102 uint8_t interactive = 0;
 103 uint8_t auto_start = 0;
 104 uint8_t tx_first;
 105 char cmdline_filename[PATH_MAX] = {0};
 106
 107 /*
 108  * NUMA support configuration.
 109  * When set, the NUMA support attempts to dispatch the allocation of the
 110  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 111  * probed ports among the CPU sockets 0 and 1.
 112  * Otherwise, all memory is allocated from CPU socket 0.
 113  */
 114 uint8_t numa_support = 1; /**< numa enabled by default */
 115
 116 /*
 117  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 118  * not configured.
 119  */
 120 uint8_t socket_num = UMA_NO_CONFIG;
 121
 122 /*
 123  * Select mempool allocation type:
 124  * - native: use regular DPDK memory
 125  * - anon: use regular DPDK memory to create mempool, but populate using
 126  *         anonymous memory (may not be IOVA-contiguous)
 127  * - xmem: use externally allocated hugepage memory
 128  */
 129 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 130
 131 /*
 132  * Store specified sockets on which memory pool to be used by ports
 133  * is allocated.
 134  */
 135 uint8_t port_numa[RTE_MAX_ETHPORTS];
 136
 137 /*
 138  * Store specified sockets on which RX ring to be used by ports
 139  * is allocated.
 140  */
 141 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 142
 143 /*
 144  * Store specified sockets on which TX ring to be used by ports
 145  * is allocated.
 146  */
 147 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 148
 149 /*
 150  * Record the Ethernet address of peer target ports to which packets are
 151  * forwarded.
 152  * Must be instantiated with the ethernet addresses of peer traffic generator
 153  * ports.
 154  */
 155 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 156 portid_t nb_peer_eth_addrs = 0;
 157
 158 /*
 159  * Probed Target Environment.
 160  */
 161 struct rte_port *ports;        /**< For all probed ethernet ports. */
 162 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 163 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 164 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 165
 166 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 167
 168 /*
 169  * Test Forwarding Configuration.
 170  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 171  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 172  */
 173 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 174 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 175 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 176 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 177
 178 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 179 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 180
 181 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 182 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 183
 184 /*
 185  * Forwarding engines.
 186  */
 187 struct fwd_engine * fwd_engines[] = {
 188         &io_fwd_engine,
 189         &mac_fwd_engine,
 190         &mac_swap_engine,
 191         &flow_gen_engine,
 192         &rx_only_engine,
 193         &tx_only_engine,
 194         &csum_fwd_engine,
 195         &icmp_echo_engine,
 196         &noisy_vnf_engine,
 197         &five_tuple_swap_fwd_engine,
 198 #ifdef RTE_LIBRTE_IEEE1588
 199         &ieee1588_fwd_engine,
 200 #endif
 201         &shared_rxq_engine,
 202         NULL,
 203 };
 204
 205 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 206 uint16_t mempool_flags;
 207
 208 struct fwd_config cur_fwd_config;
 209 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 210 uint32_t retry_enabled;
 211 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 212 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 213
 214 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 215 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 216         DEFAULT_MBUF_DATA_SIZE
 217 }; /**< Mbuf data space size. */
 218 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 219                                       * specified on command-line. */
 220 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 221
 222 /** Extended statistics to show. */
 223 struct rte_eth_xstat_name *xstats_display;
 224
 225 unsigned int xstats_display_num; /**< Size of extended statistics to show */
 226
 227 /*
 228  * In container, it cannot terminate the process which running with 'stats-period'
 229  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 230  */
 231 uint8_t f_quit;
 232
 233 /*
 234  * Max Rx frame size, set by '--max-pkt-len' parameter.
 235  */
 236 uint32_t max_rx_pkt_len;
 237
 238 /*
 239  * Configuration of packet segments used to scatter received packets
 240  * if some of split features is configured.
 241  */
 242 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 243 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 244 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 245 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 246
 247 /*
 248  * Configuration of packet segments used by the "txonly" processing engine.
 249  */
 250 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 251 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 252         TXONLY_DEF_PACKET_LEN,
 253 };
 254 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 255
 256 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 257 /**< Split policy for packets to TX. */
 258
 259 uint8_t txonly_multi_flow;
 260 /**< Whether multiple flows are generated in TXONLY mode. */
 261
 262 uint32_t tx_pkt_times_inter;
 263 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 264
 265 uint32_t tx_pkt_times_intra;
 266 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 267
 268 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 269 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 270 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
 271 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 272
 273 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 274 uint8_t dcb_config = 0;
 275
 276 /*
 277  * Configurable number of RX/TX queues.
 278  */
 279 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 280 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 281 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 282
 283 /*
 284  * Configurable number of RX/TX ring descriptors.
 285  * Defaults are supplied by drivers via ethdev.
 286  */
 287 #define RTE_TEST_RX_DESC_DEFAULT 0
 288 #define RTE_TEST_TX_DESC_DEFAULT 0
 289 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 290 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 291
 292 #define RTE_PMD_PARAM_UNSET -1
 293 /*
 294  * Configurable values of RX and TX ring threshold registers.
 295  */
 296
 297 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 298 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 299 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 300
 301 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 302 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 303 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 304
 305 /*
 306  * Configurable value of RX free threshold.
 307  */
 308 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 309
 310 /*
 311  * Configurable value of RX drop enable.
 312  */
 313 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 314
 315 /*
 316  * Configurable value of TX free threshold.
 317  */
 318 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 319
 320 /*
 321  * Configurable value of TX RS bit threshold.
 322  */
 323 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 324
 325 /*
 326  * Configurable value of buffered packets before sending.
 327  */
 328 uint16_t noisy_tx_sw_bufsz;
 329
 330 /*
 331  * Configurable value of packet buffer timeout.
 332  */
 333 uint16_t noisy_tx_sw_buf_flush_time;
 334
 335 /*
 336  * Configurable value for size of VNF internal memory area
 337  * used for simulating noisy neighbour behaviour
 338  */
 339 uint64_t noisy_lkup_mem_sz;
 340
 341 /*
 342  * Configurable value of number of random writes done in
 343  * VNF simulation memory area.
 344  */
 345 uint64_t noisy_lkup_num_writes;
 346
 347 /*
 348  * Configurable value of number of random reads done in
 349  * VNF simulation memory area.
 350  */
 351 uint64_t noisy_lkup_num_reads;
 352
 353 /*
 354  * Configurable value of number of random reads/writes done in
 355  * VNF simulation memory area.
 356  */
 357 uint64_t noisy_lkup_num_reads_writes;
 358
 359 /*
 360  * Receive Side Scaling (RSS) configuration.
 361  */
 362 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */
 363
 364 /*
 365  * Port topology configuration
 366  */
 367 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 368
 369 /*
 370  * Avoids to flush all the RX streams before starts forwarding.
 371  */
 372 uint8_t no_flush_rx = 0; /* flush by default */
 373
 374 /*
 375  * Flow API isolated mode.
 376  */
 377 uint8_t flow_isolate_all;
 378
 379 /*
 380  * Avoids to check link status when starting/stopping a port.
 381  */
 382 uint8_t no_link_check = 0; /* check by default */
 383
 384 /*
 385  * Don't automatically start all ports in interactive mode.
 386  */
 387 uint8_t no_device_start = 0;
 388
 389 /*
 390  * Enable link status change notification
 391  */
 392 uint8_t lsc_interrupt = 1; /* enabled by default */
 393
 394 /*
 395  * Enable device removal notification.
 396  */
 397 uint8_t rmv_interrupt = 1; /* enabled by default */
 398
 399 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 400
 401 /* After attach, port setup is called on event or by iterator */
 402 bool setup_on_probe_event = true;
 403
 404 /* Clear ptypes on port initialization. */
 405 uint8_t clear_ptypes = true;
 406
 407 /* Hairpin ports configuration mode. */
 408 uint16_t hairpin_mode;
 409
 410 /* Pretty printing of ethdev events */
 411 static const char * const eth_event_desc[] = {
 412         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 413         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 414         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 415         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 416         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 417         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 418         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 419         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 420         [RTE_ETH_EVENT_NEW] = "device probed",
 421         [RTE_ETH_EVENT_DESTROY] = "device released",
 422         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 423         [RTE_ETH_EVENT_RX_AVAIL_THRESH] = "RxQ available descriptors threshold reached",
 424         [RTE_ETH_EVENT_MAX] = NULL,
 425 };
 426
 427 /*
 428  * Display or mask ether events
 429  * Default to all events except VF_MBOX
 430  */
 431 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 432                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 433                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 434                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 435                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 436                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 437                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 438                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 439 /*
 440  * Decide if all memory are locked for performance.
 441  */
 442 int do_mlockall = 0;
 443
 444 /*
 445  * NIC bypass mode configuration options.
 446  */
 447
 448 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 449 /* The NIC bypass watchdog timeout. */
 450 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 451 #endif
 452
 453
 454 #ifdef RTE_LIB_LATENCYSTATS
 455
 456 /*
 457  * Set when latency stats is enabled in the commandline
 458  */
 459 uint8_t latencystats_enabled;
 460
 461 /*
 462  * Lcore ID to service latency statistics.
 463  */
 464 lcoreid_t latencystats_lcore_id = -1;
 465
 466 #endif
 467
 468 /*
 469  * Ethernet device configuration.
 470  */
 471 struct rte_eth_rxmode rx_mode;
 472
 473 struct rte_eth_txmode tx_mode = {
 474         .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
 475 };
 476
 477 struct rte_eth_fdir_conf fdir_conf = {
 478         .mode = RTE_FDIR_MODE_NONE,
 479         .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
 480         .status = RTE_FDIR_REPORT_STATUS,
 481         .mask = {
 482                 .vlan_tci_mask = 0xFFEF,
 483                 .ipv4_mask     = {
 484                         .src_ip = 0xFFFFFFFF,
 485                         .dst_ip = 0xFFFFFFFF,
 486                 },
 487                 .ipv6_mask     = {
 488                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 489                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 490                 },
 491                 .src_port_mask = 0xFFFF,
 492                 .dst_port_mask = 0xFFFF,
 493                 .mac_addr_byte_mask = 0xFF,
 494                 .tunnel_type_mask = 1,
 495                 .tunnel_id_mask = 0xFFFFFFFF,
 496         },
 497         .drop_queue = 127,
 498 };
 499
 500 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 501
 502 /*
 503  * Display zero values by default for xstats
 504  */
 505 uint8_t xstats_hide_zero;
 506
 507 /*
 508  * Measure of CPU cycles disabled by default
 509  */
 510 uint8_t record_core_cycles;
 511
 512 /*
 513  * Display of RX and TX bursts disabled by default
 514  */
 515 uint8_t record_burst_stats;
 516
 517 /*
 518  * Number of ports per shared Rx queue group, 0 disable.
 519  */
 520 uint32_t rxq_share;
 521
 522 unsigned int num_sockets = 0;
 523 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 524
 525 #ifdef RTE_LIB_BITRATESTATS
 526 /* Bitrate statistics */
 527 struct rte_stats_bitrates *bitrate_data;
 528 lcoreid_t bitrate_lcore_id;
 529 uint8_t bitrate_enabled;
 530 #endif
 531
 532 #ifdef RTE_LIB_GRO
 533 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 534 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 535 #endif
 536
 537 /*
 538  * hexadecimal bitmask of RX mq mode can be enabled.
 539  */
 540 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
 541
 542 /*
 543  * Used to set forced link speed
 544  */
 545 uint32_t eth_link_speed;
 546
 547 /*
 548  * ID of the current process in multi-process, used to
 549  * configure the queues to be polled.
 550  */
 551 int proc_id;
 552
 553 /*
 554  * Number of processes in multi-process, used to
 555  * configure the queues to be polled.
 556  */
 557 unsigned int num_procs = 1;
 558
 559 static void
 560 eth_rx_metadata_negotiate_mp(uint16_t port_id)
 561 {
 562         uint64_t rx_meta_features = 0;
 563         int ret;
 564
 565         if (!is_proc_primary())
 566                 return;
 567
 568         rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
 569         rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
 570         rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
 571
 572         ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
 573         if (ret == 0) {
 574                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
 575                         TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
 576                                     port_id);
 577                 }
 578
 579                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
 580                         TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
 581                                     port_id);
 582                 }
 583
 584                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
 585                         TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
 586                                     port_id);
 587                 }
 588         } else if (ret != -ENOTSUP) {
 589                 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
 590                          port_id, rte_strerror(-ret));
 591         }
 592 }
 593
 594 static int
 595 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 596                       const struct rte_eth_conf *dev_conf)
 597 {
 598         if (is_proc_primary())
 599                 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
 600                                         dev_conf);
 601         return 0;
 602 }
 603
 604 static int
 605 change_bonding_slave_port_status(portid_t bond_pid, bool is_stop)
 606 {
 607 #ifdef RTE_NET_BOND
 608
 609         portid_t slave_pids[RTE_MAX_ETHPORTS];
 610         struct rte_port *port;
 611         int num_slaves;
 612         portid_t slave_pid;
 613         int i;
 614
 615         num_slaves = rte_eth_bond_slaves_get(bond_pid, slave_pids,
 616                                                 RTE_MAX_ETHPORTS);
 617         if (num_slaves < 0) {
 618                 fprintf(stderr, "Failed to get slave list for port = %u\n",
 619                         bond_pid);
 620                 return num_slaves;
 621         }
 622
 623         for (i = 0; i < num_slaves; i++) {
 624                 slave_pid = slave_pids[i];
 625                 port = &ports[slave_pid];
 626                 port->port_status =
 627                         is_stop ? RTE_PORT_STOPPED : RTE_PORT_STARTED;
 628         }
 629 #else
 630         RTE_SET_USED(bond_pid);
 631         RTE_SET_USED(is_stop);
 632 #endif
 633         return 0;
 634 }
 635
 636 static int
 637 eth_dev_start_mp(uint16_t port_id)
 638 {
 639         int ret;
 640
 641         if (is_proc_primary()) {
 642                 ret = rte_eth_dev_start(port_id);
 643                 if (ret != 0)
 644                         return ret;
 645
 646                 struct rte_port *port = &ports[port_id];
 647
 648                 /*
 649                  * Starting a bonded port also starts all slaves under the bonded
 650                  * device. So if this port is bond device, we need to modify the
 651                  * port status of these slaves.
 652                  */
 653                 if (port->bond_flag == 1)
 654                         return change_bonding_slave_port_status(port_id, false);
 655         }
 656
 657         return 0;
 658 }
 659
 660 static int
 661 eth_dev_stop_mp(uint16_t port_id)
 662 {
 663         int ret;
 664
 665         if (is_proc_primary()) {
 666                 ret = rte_eth_dev_stop(port_id);
 667                 if (ret != 0)
 668                         return ret;
 669
 670                 struct rte_port *port = &ports[port_id];
 671
 672                 /*
 673                  * Stopping a bonded port also stops all slaves under the bonded
 674                  * device. So if this port is bond device, we need to modify the
 675                  * port status of these slaves.
 676                  */
 677                 if (port->bond_flag == 1)
 678                         return change_bonding_slave_port_status(port_id, true);
 679         }
 680
 681         return 0;
 682 }
 683
 684 static void
 685 mempool_free_mp(struct rte_mempool *mp)
 686 {
 687         if (is_proc_primary())
 688                 rte_mempool_free(mp);
 689 }
 690
 691 static int
 692 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
 693 {
 694         if (is_proc_primary())
 695                 return rte_eth_dev_set_mtu(port_id, mtu);
 696
 697         return 0;
 698 }
 699
 700 /* Forward function declarations */
 701 static void setup_attached_port(portid_t pi);
 702 static void check_all_ports_link_status(uint32_t port_mask);
 703 static int eth_event_callback(portid_t port_id,
 704                               enum rte_eth_event_type type,
 705                               void *param, void *ret_param);
 706 static void dev_event_callback(const char *device_name,
 707                                 enum rte_dev_event_type type,
 708                                 void *param);
 709 static void fill_xstats_display_info(void);
 710
 711 /*
 712  * Check if all the ports are started.
 713  * If yes, return positive value. If not, return zero.
 714  */
 715 static int all_ports_started(void);
 716
 717 #ifdef RTE_LIB_GSO
 718 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 719 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 720 #endif
 721
 722 /* Holds the registered mbuf dynamic flags names. */
 723 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 724
 725
 726 /*
 727  * Helper function to check if socket is already discovered.
 728  * If yes, return positive value. If not, return zero.
 729  */
 730 int
 731 new_socket_id(unsigned int socket_id)
 732 {
 733         unsigned int i;
 734
 735         for (i = 0; i < num_sockets; i++) {
 736                 if (socket_ids[i] == socket_id)
 737                         return 0;
 738         }
 739         return 1;
 740 }
 741
 742 /*
 743  * Setup default configuration.
 744  */
 745 static void
 746 set_default_fwd_lcores_config(void)
 747 {
 748         unsigned int i;
 749         unsigned int nb_lc;
 750         unsigned int sock_num;
 751
 752         nb_lc = 0;
 753         for (i = 0; i < RTE_MAX_LCORE; i++) {
 754                 if (!rte_lcore_is_enabled(i))
 755                         continue;
 756                 sock_num = rte_lcore_to_socket_id(i);
 757                 if (new_socket_id(sock_num)) {
 758                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 759                                 rte_exit(EXIT_FAILURE,
 760                                          "Total sockets greater than %u\n",
 761                                          RTE_MAX_NUMA_NODES);
 762                         }
 763                         socket_ids[num_sockets++] = sock_num;
 764                 }
 765                 if (i == rte_get_main_lcore())
 766                         continue;
 767                 fwd_lcores_cpuids[nb_lc++] = i;
 768         }
 769         nb_lcores = (lcoreid_t) nb_lc;
 770         nb_cfg_lcores = nb_lcores;
 771         nb_fwd_lcores = 1;
 772 }
 773
 774 static void
 775 set_def_peer_eth_addrs(void)
 776 {
 777         portid_t i;
 778
 779         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 780                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 781                 peer_eth_addrs[i].addr_bytes[5] = i;
 782         }
 783 }
 784
 785 static void
 786 set_default_fwd_ports_config(void)
 787 {
 788         portid_t pt_id;
 789         int i = 0;
 790
 791         RTE_ETH_FOREACH_DEV(pt_id) {
 792                 fwd_ports_ids[i++] = pt_id;
 793
 794                 /* Update sockets info according to the attached device */
 795                 int socket_id = rte_eth_dev_socket_id(pt_id);
 796                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 797                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 798                                 rte_exit(EXIT_FAILURE,
 799                                          "Total sockets greater than %u\n",
 800                                          RTE_MAX_NUMA_NODES);
 801                         }
 802                         socket_ids[num_sockets++] = socket_id;
 803                 }
 804         }
 805
 806         nb_cfg_ports = nb_ports;
 807         nb_fwd_ports = nb_ports;
 808 }
 809
 810 void
 811 set_def_fwd_config(void)
 812 {
 813         set_default_fwd_lcores_config();
 814         set_def_peer_eth_addrs();
 815         set_default_fwd_ports_config();
 816 }
 817
 818 #ifndef RTE_EXEC_ENV_WINDOWS
 819 /* extremely pessimistic estimation of memory required to create a mempool */
 820 static int
 821 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 822 {
 823         unsigned int n_pages, mbuf_per_pg, leftover;
 824         uint64_t total_mem, mbuf_mem, obj_sz;
 825
 826         /* there is no good way to predict how much space the mempool will
 827          * occupy because it will allocate chunks on the fly, and some of those
 828          * will come from default DPDK memory while some will come from our
 829          * external memory, so just assume 128MB will be enough for everyone.
 830          */
 831         uint64_t hdr_mem = 128 << 20;
 832
 833         /* account for possible non-contiguousness */
 834         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 835         if (obj_sz > pgsz) {
 836                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 837                 return -1;
 838         }
 839
 840         mbuf_per_pg = pgsz / obj_sz;
 841         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 842         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 843
 844         mbuf_mem = n_pages * pgsz;
 845
 846         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 847
 848         if (total_mem > SIZE_MAX) {
 849                 TESTPMD_LOG(ERR, "Memory size too big\n");
 850                 return -1;
 851         }
 852         *out = (size_t)total_mem;
 853
 854         return 0;
 855 }
 856
 857 static int
 858 pagesz_flags(uint64_t page_sz)
 859 {
 860         /* as per mmap() manpage, all page sizes are log2 of page size
 861          * shifted by MAP_HUGE_SHIFT
 862          */
 863         int log2 = rte_log2_u64(page_sz);
 864
 865         return (log2 << HUGE_SHIFT);
 866 }
 867
 868 static void *
 869 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 870 {
 871         void *addr;
 872         int flags;
 873
 874         /* allocate anonymous hugepages */
 875         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 876         if (huge)
 877                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 878
 879         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 880         if (addr == MAP_FAILED)
 881                 return NULL;
 882
 883         return addr;
 884 }
 885
 886 struct extmem_param {
 887         void *addr;
 888         size_t len;
 889         size_t pgsz;
 890         rte_iova_t *iova_table;
 891         unsigned int iova_table_len;
 892 };
 893
 894 static int
 895 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 896                 bool huge)
 897 {
 898         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 899                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 900         unsigned int cur_page, n_pages, pgsz_idx;
 901         size_t mem_sz, cur_pgsz;
 902         rte_iova_t *iovas = NULL;
 903         void *addr;
 904         int ret;
 905
 906         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 907                 /* skip anything that is too big */
 908                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 909                         continue;
 910
 911                 cur_pgsz = pgsizes[pgsz_idx];
 912
 913                 /* if we were told not to allocate hugepages, override */
 914                 if (!huge)
 915                         cur_pgsz = sysconf(_SC_PAGESIZE);
 916
 917                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 918                 if (ret < 0) {
 919                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 920                         return -1;
 921                 }
 922
 923                 /* allocate our memory */
 924                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 925
 926                 /* if we couldn't allocate memory with a specified page size,
 927                  * that doesn't mean we can't do it with other page sizes, so
 928                  * try another one.
 929                  */
 930                 if (addr == NULL)
 931                         continue;
 932
 933                 /* store IOVA addresses for every page in this memory area */
 934                 n_pages = mem_sz / cur_pgsz;
 935
 936                 iovas = malloc(sizeof(*iovas) * n_pages);
 937
 938                 if (iovas == NULL) {
 939                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 940                         goto fail;
 941                 }
 942                 /* lock memory if it's not huge pages */
 943                 if (!huge)
 944                         mlock(addr, mem_sz);
 945
 946                 /* populate IOVA addresses */
 947                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 948                         rte_iova_t iova;
 949                         size_t offset;
 950                         void *cur;
 951
 952                         offset = cur_pgsz * cur_page;
 953                         cur = RTE_PTR_ADD(addr, offset);
 954
 955                         /* touch the page before getting its IOVA */
 956                         *(volatile char *)cur = 0;
 957
 958                         iova = rte_mem_virt2iova(cur);
 959
 960                         iovas[cur_page] = iova;
 961                 }
 962
 963                 break;
 964         }
 965         /* if we couldn't allocate anything */
 966         if (iovas == NULL)
 967                 return -1;
 968
 969         param->addr = addr;
 970         param->len = mem_sz;
 971         param->pgsz = cur_pgsz;
 972         param->iova_table = iovas;
 973         param->iova_table_len = n_pages;
 974
 975         return 0;
 976 fail:
 977         free(iovas);
 978         if (addr)
 979                 munmap(addr, mem_sz);
 980
 981         return -1;
 982 }
 983
 984 static int
 985 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 986 {
 987         struct extmem_param param;
 988         int socket_id, ret;
 989
 990         memset(&param, 0, sizeof(param));
 991
 992         /* check if our heap exists */
 993         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 994         if (socket_id < 0) {
 995                 /* create our heap */
 996                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 997                 if (ret < 0) {
 998                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 999                         return -1;
1000                 }
1001         }
1002
1003         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
1004         if (ret < 0) {
1005                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
1006                 return -1;
1007         }
1008
1009         /* we now have a valid memory area, so add it to heap */
1010         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
1011                         param.addr, param.len, param.iova_table,
1012                         param.iova_table_len, param.pgsz);
1013
1014         /* when using VFIO, memory is automatically mapped for DMA by EAL */
1015
1016         /* not needed any more */
1017         free(param.iova_table);
1018
1019         if (ret < 0) {
1020                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
1021                 munmap(param.addr, param.len);
1022                 return -1;
1023         }
1024
1025         /* success */
1026
1027         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
1028                         param.len >> 20);
1029
1030         return 0;
1031 }
1032 static void
1033 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1034              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1035 {
1036         uint16_t pid = 0;
1037         int ret;
1038
1039         RTE_ETH_FOREACH_DEV(pid) {
1040                 struct rte_eth_dev_info dev_info;
1041
1042                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1043                 if (ret != 0) {
1044                         TESTPMD_LOG(DEBUG,
1045                                     "unable to get device info for port %d on addr 0x%p,"
1046                                     "mempool unmapping will not be performed\n",
1047                                     pid, memhdr->addr);
1048                         continue;
1049                 }
1050
1051                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
1052                 if (ret) {
1053                         TESTPMD_LOG(DEBUG,
1054                                     "unable to DMA unmap addr 0x%p "
1055                                     "for device %s\n",
1056                                     memhdr->addr, dev_info.device->name);
1057                 }
1058         }
1059         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
1060         if (ret) {
1061                 TESTPMD_LOG(DEBUG,
1062                             "unable to un-register addr 0x%p\n", memhdr->addr);
1063         }
1064 }
1065
1066 static void
1067 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
1068            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
1069 {
1070         uint16_t pid = 0;
1071         size_t page_size = sysconf(_SC_PAGESIZE);
1072         int ret;
1073
1074         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1075                                   page_size);
1076         if (ret) {
1077                 TESTPMD_LOG(DEBUG,
1078                             "unable to register addr 0x%p\n", memhdr->addr);
1079                 return;
1080         }
1081         RTE_ETH_FOREACH_DEV(pid) {
1082                 struct rte_eth_dev_info dev_info;
1083
1084                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1085                 if (ret != 0) {
1086                         TESTPMD_LOG(DEBUG,
1087                                     "unable to get device info for port %d on addr 0x%p,"
1088                                     "mempool mapping will not be performed\n",
1089                                     pid, memhdr->addr);
1090                         continue;
1091                 }
1092                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1093                 if (ret) {
1094                         TESTPMD_LOG(DEBUG,
1095                                     "unable to DMA map addr 0x%p "
1096                                     "for device %s\n",
1097                                     memhdr->addr, dev_info.device->name);
1098                 }
1099         }
1100 }
1101 #endif
1102
1103 static unsigned int
1104 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1105             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1106 {
1107         struct rte_pktmbuf_extmem *xmem;
1108         unsigned int ext_num, zone_num, elt_num;
1109         uint16_t elt_size;
1110
1111         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1112         elt_num = EXTBUF_ZONE_SIZE / elt_size;
1113         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1114
1115         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1116         if (xmem == NULL) {
1117                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1118                                  "external buffer descriptors\n");
1119                 *ext_mem = NULL;
1120                 return 0;
1121         }
1122         for (ext_num = 0; ext_num < zone_num; ext_num++) {
1123                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1124                 const struct rte_memzone *mz;
1125                 char mz_name[RTE_MEMZONE_NAMESIZE];
1126                 int ret;
1127
1128                 ret = snprintf(mz_name, sizeof(mz_name),
1129                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1130                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1131                         errno = ENAMETOOLONG;
1132                         ext_num = 0;
1133                         break;
1134                 }
1135                 mz = rte_memzone_reserve(mz_name, EXTBUF_ZONE_SIZE,
1136                                          socket_id,
1137                                          RTE_MEMZONE_IOVA_CONTIG |
1138                                          RTE_MEMZONE_1GB |
1139                                          RTE_MEMZONE_SIZE_HINT_ONLY);
1140                 if (mz == NULL) {
1141                         /*
1142                          * The caller exits on external buffer creation
1143                          * error, so there is no need to free memzones.
1144                          */
1145                         errno = ENOMEM;
1146                         ext_num = 0;
1147                         break;
1148                 }
1149                 xseg->buf_ptr = mz->addr;
1150                 xseg->buf_iova = mz->iova;
1151                 xseg->buf_len = EXTBUF_ZONE_SIZE;
1152                 xseg->elt_size = elt_size;
1153         }
1154         if (ext_num == 0 && xmem != NULL) {
1155                 free(xmem);
1156                 xmem = NULL;
1157         }
1158         *ext_mem = xmem;
1159         return ext_num;
1160 }
1161
1162 /*
1163  * Configuration initialisation done once at init time.
1164  */
1165 static struct rte_mempool *
1166 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1167                  unsigned int socket_id, uint16_t size_idx)
1168 {
1169         char pool_name[RTE_MEMPOOL_NAMESIZE];
1170         struct rte_mempool *rte_mp = NULL;
1171 #ifndef RTE_EXEC_ENV_WINDOWS
1172         uint32_t mb_size;
1173
1174         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1175 #endif
1176         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1177         if (!is_proc_primary()) {
1178                 rte_mp = rte_mempool_lookup(pool_name);
1179                 if (rte_mp == NULL)
1180                         rte_exit(EXIT_FAILURE,
1181                                 "Get mbuf pool for socket %u failed: %s\n",
1182                                 socket_id, rte_strerror(rte_errno));
1183                 return rte_mp;
1184         }
1185
1186         TESTPMD_LOG(INFO,
1187                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1188                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1189
1190         switch (mp_alloc_type) {
1191         case MP_ALLOC_NATIVE:
1192                 {
1193                         /* wrapper to rte_mempool_create() */
1194                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1195                                         rte_mbuf_best_mempool_ops());
1196                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1197                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1198                         break;
1199                 }
1200 #ifndef RTE_EXEC_ENV_WINDOWS
1201         case MP_ALLOC_ANON:
1202                 {
1203                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1204                                 mb_size, (unsigned int) mb_mempool_cache,
1205                                 sizeof(struct rte_pktmbuf_pool_private),
1206                                 socket_id, mempool_flags);
1207                         if (rte_mp == NULL)
1208                                 goto err;
1209
1210                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1211                                 rte_mempool_free(rte_mp);
1212                                 rte_mp = NULL;
1213                                 goto err;
1214                         }
1215                         rte_pktmbuf_pool_init(rte_mp, NULL);
1216                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1217                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1218                         break;
1219                 }
1220         case MP_ALLOC_XMEM:
1221         case MP_ALLOC_XMEM_HUGE:
1222                 {
1223                         int heap_socket;
1224                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1225
1226                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1227                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1228
1229                         heap_socket =
1230                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1231                         if (heap_socket < 0)
1232                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1233
1234                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1235                                         rte_mbuf_best_mempool_ops());
1236                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1237                                         mb_mempool_cache, 0, mbuf_seg_size,
1238                                         heap_socket);
1239                         break;
1240                 }
1241 #endif
1242         case MP_ALLOC_XBUF:
1243                 {
1244                         struct rte_pktmbuf_extmem *ext_mem;
1245                         unsigned int ext_num;
1246
1247                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1248                                                socket_id, pool_name, &ext_mem);
1249                         if (ext_num == 0)
1250                                 rte_exit(EXIT_FAILURE,
1251                                          "Can't create pinned data buffers\n");
1252
1253                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1254                                         rte_mbuf_best_mempool_ops());
1255                         rte_mp = rte_pktmbuf_pool_create_extbuf
1256                                         (pool_name, nb_mbuf, mb_mempool_cache,
1257                                          0, mbuf_seg_size, socket_id,
1258                                          ext_mem, ext_num);
1259                         free(ext_mem);
1260                         break;
1261                 }
1262         default:
1263                 {
1264                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1265                 }
1266         }
1267
1268 #ifndef RTE_EXEC_ENV_WINDOWS
1269 err:
1270 #endif
1271         if (rte_mp == NULL) {
1272                 rte_exit(EXIT_FAILURE,
1273                         "Creation of mbuf pool for socket %u failed: %s\n",
1274                         socket_id, rte_strerror(rte_errno));
1275         } else if (verbose_level > 0) {
1276                 rte_mempool_dump(stdout, rte_mp);
1277         }
1278         return rte_mp;
1279 }
1280
1281 /*
1282  * Check given socket id is valid or not with NUMA mode,
1283  * if valid, return 0, else return -1
1284  */
1285 static int
1286 check_socket_id(const unsigned int socket_id)
1287 {
1288         static int warning_once = 0;
1289
1290         if (new_socket_id(socket_id)) {
1291                 if (!warning_once && numa_support)
1292                         fprintf(stderr,
1293                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1294                 warning_once = 1;
1295                 return -1;
1296         }
1297         return 0;
1298 }
1299
1300 /*
1301  * Get the allowed maximum number of RX queues.
1302  * *pid return the port id which has minimal value of
1303  * max_rx_queues in all ports.
1304  */
1305 queueid_t
1306 get_allowed_max_nb_rxq(portid_t *pid)
1307 {
1308         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1309         bool max_rxq_valid = false;
1310         portid_t pi;
1311         struct rte_eth_dev_info dev_info;
1312
1313         RTE_ETH_FOREACH_DEV(pi) {
1314                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1315                         continue;
1316
1317                 max_rxq_valid = true;
1318                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1319                         allowed_max_rxq = dev_info.max_rx_queues;
1320                         *pid = pi;
1321                 }
1322         }
1323         return max_rxq_valid ? allowed_max_rxq : 0;
1324 }
1325
1326 /*
1327  * Check input rxq is valid or not.
1328  * If input rxq is not greater than any of maximum number
1329  * of RX queues of all ports, it is valid.
1330  * if valid, return 0, else return -1
1331  */
1332 int
1333 check_nb_rxq(queueid_t rxq)
1334 {
1335         queueid_t allowed_max_rxq;
1336         portid_t pid = 0;
1337
1338         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1339         if (rxq > allowed_max_rxq) {
1340                 fprintf(stderr,
1341                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1342                         rxq, allowed_max_rxq, pid);
1343                 return -1;
1344         }
1345         return 0;
1346 }
1347
1348 /*
1349  * Get the allowed maximum number of TX queues.
1350  * *pid return the port id which has minimal value of
1351  * max_tx_queues in all ports.
1352  */
1353 queueid_t
1354 get_allowed_max_nb_txq(portid_t *pid)
1355 {
1356         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1357         bool max_txq_valid = false;
1358         portid_t pi;
1359         struct rte_eth_dev_info dev_info;
1360
1361         RTE_ETH_FOREACH_DEV(pi) {
1362                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1363                         continue;
1364
1365                 max_txq_valid = true;
1366                 if (dev_info.max_tx_queues < allowed_max_txq) {
1367                         allowed_max_txq = dev_info.max_tx_queues;
1368                         *pid = pi;
1369                 }
1370         }
1371         return max_txq_valid ? allowed_max_txq : 0;
1372 }
1373
1374 /*
1375  * Check input txq is valid or not.
1376  * If input txq is not greater than any of maximum number
1377  * of TX queues of all ports, it is valid.
1378  * if valid, return 0, else return -1
1379  */
1380 int
1381 check_nb_txq(queueid_t txq)
1382 {
1383         queueid_t allowed_max_txq;
1384         portid_t pid = 0;
1385
1386         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1387         if (txq > allowed_max_txq) {
1388                 fprintf(stderr,
1389                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1390                         txq, allowed_max_txq, pid);
1391                 return -1;
1392         }
1393         return 0;
1394 }
1395
1396 /*
1397  * Get the allowed maximum number of RXDs of every rx queue.
1398  * *pid return the port id which has minimal value of
1399  * max_rxd in all queues of all ports.
1400  */
1401 static uint16_t
1402 get_allowed_max_nb_rxd(portid_t *pid)
1403 {
1404         uint16_t allowed_max_rxd = UINT16_MAX;
1405         portid_t pi;
1406         struct rte_eth_dev_info dev_info;
1407
1408         RTE_ETH_FOREACH_DEV(pi) {
1409                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1410                         continue;
1411
1412                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1413                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1414                         *pid = pi;
1415                 }
1416         }
1417         return allowed_max_rxd;
1418 }
1419
1420 /*
1421  * Get the allowed minimal number of RXDs of every rx queue.
1422  * *pid return the port id which has minimal value of
1423  * min_rxd in all queues of all ports.
1424  */
1425 static uint16_t
1426 get_allowed_min_nb_rxd(portid_t *pid)
1427 {
1428         uint16_t allowed_min_rxd = 0;
1429         portid_t pi;
1430         struct rte_eth_dev_info dev_info;
1431
1432         RTE_ETH_FOREACH_DEV(pi) {
1433                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1434                         continue;
1435
1436                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1437                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1438                         *pid = pi;
1439                 }
1440         }
1441
1442         return allowed_min_rxd;
1443 }
1444
1445 /*
1446  * Check input rxd is valid or not.
1447  * If input rxd is not greater than any of maximum number
1448  * of RXDs of every Rx queues and is not less than any of
1449  * minimal number of RXDs of every Rx queues, it is valid.
1450  * if valid, return 0, else return -1
1451  */
1452 int
1453 check_nb_rxd(queueid_t rxd)
1454 {
1455         uint16_t allowed_max_rxd;
1456         uint16_t allowed_min_rxd;
1457         portid_t pid = 0;
1458
1459         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1460         if (rxd > allowed_max_rxd) {
1461                 fprintf(stderr,
1462                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1463                         rxd, allowed_max_rxd, pid);
1464                 return -1;
1465         }
1466
1467         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1468         if (rxd < allowed_min_rxd) {
1469                 fprintf(stderr,
1470                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1471                         rxd, allowed_min_rxd, pid);
1472                 return -1;
1473         }
1474
1475         return 0;
1476 }
1477
1478 /*
1479  * Get the allowed maximum number of TXDs of every rx queues.
1480  * *pid return the port id which has minimal value of
1481  * max_txd in every tx queue.
1482  */
1483 static uint16_t
1484 get_allowed_max_nb_txd(portid_t *pid)
1485 {
1486         uint16_t allowed_max_txd = UINT16_MAX;
1487         portid_t pi;
1488         struct rte_eth_dev_info dev_info;
1489
1490         RTE_ETH_FOREACH_DEV(pi) {
1491                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1492                         continue;
1493
1494                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1495                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1496                         *pid = pi;
1497                 }
1498         }
1499         return allowed_max_txd;
1500 }
1501
1502 /*
1503  * Get the allowed maximum number of TXDs of every tx queues.
1504  * *pid return the port id which has minimal value of
1505  * min_txd in every tx queue.
1506  */
1507 static uint16_t
1508 get_allowed_min_nb_txd(portid_t *pid)
1509 {
1510         uint16_t allowed_min_txd = 0;
1511         portid_t pi;
1512         struct rte_eth_dev_info dev_info;
1513
1514         RTE_ETH_FOREACH_DEV(pi) {
1515                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1516                         continue;
1517
1518                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1519                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1520                         *pid = pi;
1521                 }
1522         }
1523
1524         return allowed_min_txd;
1525 }
1526
1527 /*
1528  * Check input txd is valid or not.
1529  * If input txd is not greater than any of maximum number
1530  * of TXDs of every Rx queues, it is valid.
1531  * if valid, return 0, else return -1
1532  */
1533 int
1534 check_nb_txd(queueid_t txd)
1535 {
1536         uint16_t allowed_max_txd;
1537         uint16_t allowed_min_txd;
1538         portid_t pid = 0;
1539
1540         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1541         if (txd > allowed_max_txd) {
1542                 fprintf(stderr,
1543                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1544                         txd, allowed_max_txd, pid);
1545                 return -1;
1546         }
1547
1548         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1549         if (txd < allowed_min_txd) {
1550                 fprintf(stderr,
1551                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1552                         txd, allowed_min_txd, pid);
1553                 return -1;
1554         }
1555         return 0;
1556 }
1557
1558
1559 /*
1560  * Get the allowed maximum number of hairpin queues.
1561  * *pid return the port id which has minimal value of
1562  * max_hairpin_queues in all ports.
1563  */
1564 queueid_t
1565 get_allowed_max_nb_hairpinq(portid_t *pid)
1566 {
1567         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1568         portid_t pi;
1569         struct rte_eth_hairpin_cap cap;
1570
1571         RTE_ETH_FOREACH_DEV(pi) {
1572                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1573                         *pid = pi;
1574                         return 0;
1575                 }
1576                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1577                         allowed_max_hairpinq = cap.max_nb_queues;
1578                         *pid = pi;
1579                 }
1580         }
1581         return allowed_max_hairpinq;
1582 }
1583
1584 /*
1585  * Check input hairpin is valid or not.
1586  * If input hairpin is not greater than any of maximum number
1587  * of hairpin queues of all ports, it is valid.
1588  * if valid, return 0, else return -1
1589  */
1590 int
1591 check_nb_hairpinq(queueid_t hairpinq)
1592 {
1593         queueid_t allowed_max_hairpinq;
1594         portid_t pid = 0;
1595
1596         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1597         if (hairpinq > allowed_max_hairpinq) {
1598                 fprintf(stderr,
1599                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1600                         hairpinq, allowed_max_hairpinq, pid);
1601                 return -1;
1602         }
1603         return 0;
1604 }
1605
1606 static int
1607 get_eth_overhead(struct rte_eth_dev_info *dev_info)
1608 {
1609         uint32_t eth_overhead;
1610
1611         if (dev_info->max_mtu != UINT16_MAX &&
1612             dev_info->max_rx_pktlen > dev_info->max_mtu)
1613                 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1614         else
1615                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1616
1617         return eth_overhead;
1618 }
1619
1620 static void
1621 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1622 {
1623         struct rte_port *port = &ports[pid];
1624         int ret;
1625         int i;
1626
1627         eth_rx_metadata_negotiate_mp(pid);
1628
1629         port->dev_conf.txmode = tx_mode;
1630         port->dev_conf.rxmode = rx_mode;
1631
1632         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1633         if (ret != 0)
1634                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1635
1636         if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1637                 port->dev_conf.txmode.offloads &=
1638                         ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1639
1640         /* Apply Rx offloads configuration */
1641         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1642                 port->rxq[i].conf.offloads = port->dev_conf.rxmode.offloads;
1643         /* Apply Tx offloads configuration */
1644         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1645                 port->txq[i].conf.offloads = port->dev_conf.txmode.offloads;
1646
1647         if (eth_link_speed)
1648                 port->dev_conf.link_speeds = eth_link_speed;
1649
1650         if (max_rx_pkt_len)
1651                 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1652                         get_eth_overhead(&port->dev_info);
1653
1654         /* set flag to initialize port/queue */
1655         port->need_reconfig = 1;
1656         port->need_reconfig_queues = 1;
1657         port->socket_id = socket_id;
1658         port->tx_metadata = 0;
1659
1660         /*
1661          * Check for maximum number of segments per MTU.
1662          * Accordingly update the mbuf data size.
1663          */
1664         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1665             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1666                 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1667                 uint16_t mtu;
1668
1669                 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1670                         uint16_t data_size = (mtu + eth_overhead) /
1671                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1672                         uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1673
1674                         if (buffer_size > mbuf_data_size[0]) {
1675                                 mbuf_data_size[0] = buffer_size;
1676                                 TESTPMD_LOG(WARNING,
1677                                         "Configured mbuf size of the first segment %hu\n",
1678                                         mbuf_data_size[0]);
1679                         }
1680                 }
1681         }
1682 }
1683
1684 static void
1685 init_config(void)
1686 {
1687         portid_t pid;
1688         struct rte_mempool *mbp;
1689         unsigned int nb_mbuf_per_pool;
1690         lcoreid_t  lc_id;
1691 #ifdef RTE_LIB_GRO
1692         struct rte_gro_param gro_param;
1693 #endif
1694 #ifdef RTE_LIB_GSO
1695         uint32_t gso_types;
1696 #endif
1697
1698         /* Configuration of logical cores. */
1699         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1700                                 sizeof(struct fwd_lcore *) * nb_lcores,
1701                                 RTE_CACHE_LINE_SIZE);
1702         if (fwd_lcores == NULL) {
1703                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1704                                                         "failed\n", nb_lcores);
1705         }
1706         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1707                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1708                                                sizeof(struct fwd_lcore),
1709                                                RTE_CACHE_LINE_SIZE);
1710                 if (fwd_lcores[lc_id] == NULL) {
1711                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1712                                                                 "failed\n");
1713                 }
1714                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1715         }
1716
1717         RTE_ETH_FOREACH_DEV(pid) {
1718                 uint32_t socket_id;
1719
1720                 if (numa_support) {
1721                         socket_id = port_numa[pid];
1722                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1723                                 socket_id = rte_eth_dev_socket_id(pid);
1724
1725                                 /*
1726                                  * if socket_id is invalid,
1727                                  * set to the first available socket.
1728                                  */
1729                                 if (check_socket_id(socket_id) < 0)
1730                                         socket_id = socket_ids[0];
1731                         }
1732                 } else {
1733                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1734                                     0 : socket_num;
1735                 }
1736                 /* Apply default TxRx configuration for all ports */
1737                 init_config_port_offloads(pid, socket_id);
1738         }
1739         /*
1740          * Create pools of mbuf.
1741          * If NUMA support is disabled, create a single pool of mbuf in
1742          * socket 0 memory by default.
1743          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1744          *
1745          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1746          * nb_txd can be configured at run time.
1747          */
1748         if (param_total_num_mbufs)
1749                 nb_mbuf_per_pool = param_total_num_mbufs;
1750         else {
1751                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1752                         (nb_lcores * mb_mempool_cache) +
1753                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1754                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1755         }
1756
1757         if (numa_support) {
1758                 uint8_t i, j;
1759
1760                 for (i = 0; i < num_sockets; i++)
1761                         for (j = 0; j < mbuf_data_size_n; j++)
1762                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1763                                         mbuf_pool_create(mbuf_data_size[j],
1764                                                           nb_mbuf_per_pool,
1765                                                           socket_ids[i], j);
1766         } else {
1767                 uint8_t i;
1768
1769                 for (i = 0; i < mbuf_data_size_n; i++)
1770                         mempools[i] = mbuf_pool_create
1771                                         (mbuf_data_size[i],
1772                                          nb_mbuf_per_pool,
1773                                          socket_num == UMA_NO_CONFIG ?
1774                                          0 : socket_num, i);
1775         }
1776
1777         init_port_config();
1778
1779 #ifdef RTE_LIB_GSO
1780         gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1781                 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1782 #endif
1783         /*
1784          * Records which Mbuf pool to use by each logical core, if needed.
1785          */
1786         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1787                 mbp = mbuf_pool_find(
1788                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1789
1790                 if (mbp == NULL)
1791                         mbp = mbuf_pool_find(0, 0);
1792                 fwd_lcores[lc_id]->mbp = mbp;
1793 #ifdef RTE_LIB_GSO
1794                 /* initialize GSO context */
1795                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1796                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1797                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1798                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1799                         RTE_ETHER_CRC_LEN;
1800                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1801 #endif
1802         }
1803
1804         fwd_config_setup();
1805
1806 #ifdef RTE_LIB_GRO
1807         /* create a gro context for each lcore */
1808         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1809         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1810         gro_param.max_item_per_flow = MAX_PKT_BURST;
1811         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1812                 gro_param.socket_id = rte_lcore_to_socket_id(
1813                                 fwd_lcores_cpuids[lc_id]);
1814                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1815                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1816                         rte_exit(EXIT_FAILURE,
1817                                         "rte_gro_ctx_create() failed\n");
1818                 }
1819         }
1820 #endif
1821 }
1822
1823
1824 void
1825 reconfig(portid_t new_port_id, unsigned socket_id)
1826 {
1827         /* Reconfiguration of Ethernet ports. */
1828         init_config_port_offloads(new_port_id, socket_id);
1829         init_port_config();
1830 }
1831
1832 int
1833 init_fwd_streams(void)
1834 {
1835         portid_t pid;
1836         struct rte_port *port;
1837         streamid_t sm_id, nb_fwd_streams_new;
1838         queueid_t q;
1839
1840         /* set socket id according to numa or not */
1841         RTE_ETH_FOREACH_DEV(pid) {
1842                 port = &ports[pid];
1843                 if (nb_rxq > port->dev_info.max_rx_queues) {
1844                         fprintf(stderr,
1845                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1846                                 nb_rxq, port->dev_info.max_rx_queues);
1847                         return -1;
1848                 }
1849                 if (nb_txq > port->dev_info.max_tx_queues) {
1850                         fprintf(stderr,
1851                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1852                                 nb_txq, port->dev_info.max_tx_queues);
1853                         return -1;
1854                 }
1855                 if (numa_support) {
1856                         if (port_numa[pid] != NUMA_NO_CONFIG)
1857                                 port->socket_id = port_numa[pid];
1858                         else {
1859                                 port->socket_id = rte_eth_dev_socket_id(pid);
1860
1861                                 /*
1862                                  * if socket_id is invalid,
1863                                  * set to the first available socket.
1864                                  */
1865                                 if (check_socket_id(port->socket_id) < 0)
1866                                         port->socket_id = socket_ids[0];
1867                         }
1868                 }
1869                 else {
1870                         if (socket_num == UMA_NO_CONFIG)
1871                                 port->socket_id = 0;
1872                         else
1873                                 port->socket_id = socket_num;
1874                 }
1875         }
1876
1877         q = RTE_MAX(nb_rxq, nb_txq);
1878         if (q == 0) {
1879                 fprintf(stderr,
1880                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1881                 return -1;
1882         }
1883         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1884         if (nb_fwd_streams_new == nb_fwd_streams)
1885                 return 0;
1886         /* clear the old */
1887         if (fwd_streams != NULL) {
1888                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1889                         if (fwd_streams[sm_id] == NULL)
1890                                 continue;
1891                         rte_free(fwd_streams[sm_id]);
1892                         fwd_streams[sm_id] = NULL;
1893                 }
1894                 rte_free(fwd_streams);
1895                 fwd_streams = NULL;
1896         }
1897
1898         /* init new */
1899         nb_fwd_streams = nb_fwd_streams_new;
1900         if (nb_fwd_streams) {
1901                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1902                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1903                         RTE_CACHE_LINE_SIZE);
1904                 if (fwd_streams == NULL)
1905                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1906                                  " (struct fwd_stream *)) failed\n",
1907                                  nb_fwd_streams);
1908
1909                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1910                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1911                                 " struct fwd_stream", sizeof(struct fwd_stream),
1912                                 RTE_CACHE_LINE_SIZE);
1913                         if (fwd_streams[sm_id] == NULL)
1914                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1915                                          "(struct fwd_stream) failed\n");
1916                 }
1917         }
1918
1919         return 0;
1920 }
1921
1922 static void
1923 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1924 {
1925         uint64_t total_burst, sburst;
1926         uint64_t nb_burst;
1927         uint64_t burst_stats[4];
1928         uint16_t pktnb_stats[4];
1929         uint16_t nb_pkt;
1930         int burst_percent[4], sburstp;
1931         int i;
1932
1933         /*
1934          * First compute the total number of packet bursts and the
1935          * two highest numbers of bursts of the same number of packets.
1936          */
1937         memset(&burst_stats, 0x0, sizeof(burst_stats));
1938         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1939
1940         /* Show stats for 0 burst size always */
1941         total_burst = pbs->pkt_burst_spread[0];
1942         burst_stats[0] = pbs->pkt_burst_spread[0];
1943         pktnb_stats[0] = 0;
1944
1945         /* Find the next 2 burst sizes with highest occurrences. */
1946         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1947                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1948
1949                 if (nb_burst == 0)
1950                         continue;
1951
1952                 total_burst += nb_burst;
1953
1954                 if (nb_burst > burst_stats[1]) {
1955                         burst_stats[2] = burst_stats[1];
1956                         pktnb_stats[2] = pktnb_stats[1];
1957                         burst_stats[1] = nb_burst;
1958                         pktnb_stats[1] = nb_pkt;
1959                 } else if (nb_burst > burst_stats[2]) {
1960                         burst_stats[2] = nb_burst;
1961                         pktnb_stats[2] = nb_pkt;
1962                 }
1963         }
1964         if (total_burst == 0)
1965                 return;
1966
1967         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1968         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1969                 if (i == 3) {
1970                         printf("%d%% of other]\n", 100 - sburstp);
1971                         return;
1972                 }
1973
1974                 sburst += burst_stats[i];
1975                 if (sburst == total_burst) {
1976                         printf("%d%% of %d pkts]\n",
1977                                 100 - sburstp, (int) pktnb_stats[i]);
1978                         return;
1979                 }
1980
1981                 burst_percent[i] =
1982                         (double)burst_stats[i] / total_burst * 100;
1983                 printf("%d%% of %d pkts + ",
1984                         burst_percent[i], (int) pktnb_stats[i]);
1985                 sburstp += burst_percent[i];
1986         }
1987 }
1988
1989 static void
1990 fwd_stream_stats_display(streamid_t stream_id)
1991 {
1992         struct fwd_stream *fs;
1993         static const char *fwd_top_stats_border = "-------";
1994
1995         fs = fwd_streams[stream_id];
1996         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1997             (fs->fwd_dropped == 0))
1998                 return;
1999         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
2000                "TX Port=%2d/Queue=%2d %s\n",
2001                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
2002                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
2003         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
2004                " TX-dropped: %-14"PRIu64,
2005                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
2006
2007         /* if checksum mode */
2008         if (cur_fwd_eng == &csum_fwd_engine) {
2009                 printf("  RX- bad IP checksum: %-14"PRIu64
2010                        "  Rx- bad L4 checksum: %-14"PRIu64
2011                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
2012                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
2013                         fs->rx_bad_outer_l4_csum);
2014                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
2015                         fs->rx_bad_outer_ip_csum);
2016         } else {
2017                 printf("\n");
2018         }
2019
2020         if (record_burst_stats) {
2021                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
2022                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
2023         }
2024 }
2025
2026 void
2027 fwd_stats_display(void)
2028 {
2029         static const char *fwd_stats_border = "----------------------";
2030         static const char *acc_stats_border = "+++++++++++++++";
2031         struct {
2032                 struct fwd_stream *rx_stream;
2033                 struct fwd_stream *tx_stream;
2034                 uint64_t tx_dropped;
2035                 uint64_t rx_bad_ip_csum;
2036                 uint64_t rx_bad_l4_csum;
2037                 uint64_t rx_bad_outer_l4_csum;
2038                 uint64_t rx_bad_outer_ip_csum;
2039         } ports_stats[RTE_MAX_ETHPORTS];
2040         uint64_t total_rx_dropped = 0;
2041         uint64_t total_tx_dropped = 0;
2042         uint64_t total_rx_nombuf = 0;
2043         struct rte_eth_stats stats;
2044         uint64_t fwd_cycles = 0;
2045         uint64_t total_recv = 0;
2046         uint64_t total_xmit = 0;
2047         struct rte_port *port;
2048         streamid_t sm_id;
2049         portid_t pt_id;
2050         int ret;
2051         int i;
2052
2053         memset(ports_stats, 0, sizeof(ports_stats));
2054
2055         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2056                 struct fwd_stream *fs = fwd_streams[sm_id];
2057
2058                 if (cur_fwd_config.nb_fwd_streams >
2059                     cur_fwd_config.nb_fwd_ports) {
2060                         fwd_stream_stats_display(sm_id);
2061                 } else {
2062                         ports_stats[fs->tx_port].tx_stream = fs;
2063                         ports_stats[fs->rx_port].rx_stream = fs;
2064                 }
2065
2066                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
2067
2068                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
2069                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2070                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2071                                 fs->rx_bad_outer_l4_csum;
2072                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2073                                 fs->rx_bad_outer_ip_csum;
2074
2075                 if (record_core_cycles)
2076                         fwd_cycles += fs->core_cycles;
2077         }
2078         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2079                 pt_id = fwd_ports_ids[i];
2080                 port = &ports[pt_id];
2081
2082                 ret = rte_eth_stats_get(pt_id, &stats);
2083                 if (ret != 0) {
2084                         fprintf(stderr,
2085                                 "%s: Error: failed to get stats (port %u): %d",
2086                                 __func__, pt_id, ret);
2087                         continue;
2088                 }
2089                 stats.ipackets -= port->stats.ipackets;
2090                 stats.opackets -= port->stats.opackets;
2091                 stats.ibytes -= port->stats.ibytes;
2092                 stats.obytes -= port->stats.obytes;
2093                 stats.imissed -= port->stats.imissed;
2094                 stats.oerrors -= port->stats.oerrors;
2095                 stats.rx_nombuf -= port->stats.rx_nombuf;
2096
2097                 total_recv += stats.ipackets;
2098                 total_xmit += stats.opackets;
2099                 total_rx_dropped += stats.imissed;
2100                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2101                 total_tx_dropped += stats.oerrors;
2102                 total_rx_nombuf  += stats.rx_nombuf;
2103
2104                 printf("\n  %s Forward statistics for port %-2d %s\n",
2105                        fwd_stats_border, pt_id, fwd_stats_border);
2106
2107                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2108                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2109                        stats.ipackets + stats.imissed);
2110
2111                 if (cur_fwd_eng == &csum_fwd_engine) {
2112                         printf("  Bad-ipcsum: %-14"PRIu64
2113                                " Bad-l4csum: %-14"PRIu64
2114                                "Bad-outer-l4csum: %-14"PRIu64"\n",
2115                                ports_stats[pt_id].rx_bad_ip_csum,
2116                                ports_stats[pt_id].rx_bad_l4_csum,
2117                                ports_stats[pt_id].rx_bad_outer_l4_csum);
2118                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
2119                                ports_stats[pt_id].rx_bad_outer_ip_csum);
2120                 }
2121                 if (stats.ierrors + stats.rx_nombuf > 0) {
2122                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
2123                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2124                 }
2125
2126                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2127                        "TX-total: %-"PRIu64"\n",
2128                        stats.opackets, ports_stats[pt_id].tx_dropped,
2129                        stats.opackets + ports_stats[pt_id].tx_dropped);
2130
2131                 if (record_burst_stats) {
2132                         if (ports_stats[pt_id].rx_stream)
2133                                 pkt_burst_stats_display("RX",
2134                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
2135                         if (ports_stats[pt_id].tx_stream)
2136                                 pkt_burst_stats_display("TX",
2137                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2138                 }
2139
2140                 printf("  %s--------------------------------%s\n",
2141                        fwd_stats_border, fwd_stats_border);
2142         }
2143
2144         printf("\n  %s Accumulated forward statistics for all ports"
2145                "%s\n",
2146                acc_stats_border, acc_stats_border);
2147         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2148                "%-"PRIu64"\n"
2149                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2150                "%-"PRIu64"\n",
2151                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2152                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2153         if (total_rx_nombuf > 0)
2154                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2155         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
2156                "%s\n",
2157                acc_stats_border, acc_stats_border);
2158         if (record_core_cycles) {
2159 #define CYC_PER_MHZ 1E6
2160                 if (total_recv > 0 || total_xmit > 0) {
2161                         uint64_t total_pkts = 0;
2162                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2163                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2164                                 total_pkts = total_xmit;
2165                         else
2166                                 total_pkts = total_recv;
2167
2168                         printf("\n  CPU cycles/packet=%.2F (total cycles="
2169                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2170                                " MHz Clock\n",
2171                                (double) fwd_cycles / total_pkts,
2172                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2173                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2174                 }
2175         }
2176 }
2177
2178 void
2179 fwd_stats_reset(void)
2180 {
2181         streamid_t sm_id;
2182         portid_t pt_id;
2183         int ret;
2184         int i;
2185
2186         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2187                 pt_id = fwd_ports_ids[i];
2188                 ret = rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2189                 if (ret != 0)
2190                         fprintf(stderr,
2191                                 "%s: Error: failed to clear stats (port %u):%d",
2192                                 __func__, pt_id, ret);
2193         }
2194         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2195                 struct fwd_stream *fs = fwd_streams[sm_id];
2196
2197                 fs->rx_packets = 0;
2198                 fs->tx_packets = 0;
2199                 fs->fwd_dropped = 0;
2200                 fs->rx_bad_ip_csum = 0;
2201                 fs->rx_bad_l4_csum = 0;
2202                 fs->rx_bad_outer_l4_csum = 0;
2203                 fs->rx_bad_outer_ip_csum = 0;
2204
2205                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2206                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2207                 fs->core_cycles = 0;
2208         }
2209 }
2210
2211 static void
2212 flush_fwd_rx_queues(void)
2213 {
2214         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2215         portid_t  rxp;
2216         portid_t port_id;
2217         queueid_t rxq;
2218         uint16_t  nb_rx;
2219         uint16_t  i;
2220         uint8_t   j;
2221         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2222         uint64_t timer_period;
2223
2224         if (num_procs > 1) {
2225                 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2226                 return;
2227         }
2228
2229         /* convert to number of cycles */
2230         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2231
2232         for (j = 0; j < 2; j++) {
2233                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2234                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2235                                 port_id = fwd_ports_ids[rxp];
2236
2237                                 /* Polling stopped queues is prohibited. */
2238                                 if (ports[port_id].rxq[rxq].state ==
2239                                     RTE_ETH_QUEUE_STATE_STOPPED)
2240                                         continue;
2241
2242                                 /**
2243                                 * testpmd can stuck in the below do while loop
2244                                 * if rte_eth_rx_burst() always returns nonzero
2245                                 * packets. So timer is added to exit this loop
2246                                 * after 1sec timer expiry.
2247                                 */
2248                                 prev_tsc = rte_rdtsc();
2249                                 do {
2250                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2251                                                 pkts_burst, MAX_PKT_BURST);
2252                                         for (i = 0; i < nb_rx; i++)
2253                                                 rte_pktmbuf_free(pkts_burst[i]);
2254
2255                                         cur_tsc = rte_rdtsc();
2256                                         diff_tsc = cur_tsc - prev_tsc;
2257                                         timer_tsc += diff_tsc;
2258                                 } while ((nb_rx > 0) &&
2259                                         (timer_tsc < timer_period));
2260                                 timer_tsc = 0;
2261                         }
2262                 }
2263                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2264         }
2265 }
2266
2267 static void
2268 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2269 {
2270         struct fwd_stream **fsm;
2271         streamid_t nb_fs;
2272         streamid_t sm_id;
2273 #ifdef RTE_LIB_BITRATESTATS
2274         uint64_t tics_per_1sec;
2275         uint64_t tics_datum;
2276         uint64_t tics_current;
2277         uint16_t i, cnt_ports;
2278
2279         cnt_ports = nb_ports;
2280         tics_datum = rte_rdtsc();
2281         tics_per_1sec = rte_get_timer_hz();
2282 #endif
2283         fsm = &fwd_streams[fc->stream_idx];
2284         nb_fs = fc->stream_nb;
2285         do {
2286                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2287                         if (!fsm[sm_id]->disabled)
2288                                 (*pkt_fwd)(fsm[sm_id]);
2289 #ifdef RTE_LIB_BITRATESTATS
2290                 if (bitrate_enabled != 0 &&
2291                                 bitrate_lcore_id == rte_lcore_id()) {
2292                         tics_current = rte_rdtsc();
2293                         if (tics_current - tics_datum >= tics_per_1sec) {
2294                                 /* Periodic bitrate calculation */
2295                                 for (i = 0; i < cnt_ports; i++)
2296                                         rte_stats_bitrate_calc(bitrate_data,
2297                                                 ports_ids[i]);
2298                                 tics_datum = tics_current;
2299                         }
2300                 }
2301 #endif
2302 #ifdef RTE_LIB_LATENCYSTATS
2303                 if (latencystats_enabled != 0 &&
2304                                 latencystats_lcore_id == rte_lcore_id())
2305                         rte_latencystats_update();
2306 #endif
2307
2308         } while (! fc->stopped);
2309 }
2310
2311 static int
2312 start_pkt_forward_on_core(void *fwd_arg)
2313 {
2314         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2315                              cur_fwd_config.fwd_eng->packet_fwd);
2316         return 0;
2317 }
2318
2319 /*
2320  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2321  * Used to start communication flows in network loopback test configurations.
2322  */
2323 static int
2324 run_one_txonly_burst_on_core(void *fwd_arg)
2325 {
2326         struct fwd_lcore *fwd_lc;
2327         struct fwd_lcore tmp_lcore;
2328
2329         fwd_lc = (struct fwd_lcore *) fwd_arg;
2330         tmp_lcore = *fwd_lc;
2331         tmp_lcore.stopped = 1;
2332         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2333         return 0;
2334 }
2335
2336 /*
2337  * Launch packet forwarding:
2338  *     - Setup per-port forwarding context.
2339  *     - launch logical cores with their forwarding configuration.
2340  */
2341 static void
2342 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2343 {
2344         unsigned int i;
2345         unsigned int lc_id;
2346         int diag;
2347
2348         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2349                 lc_id = fwd_lcores_cpuids[i];
2350                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2351                         fwd_lcores[i]->stopped = 0;
2352                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2353                                                      fwd_lcores[i], lc_id);
2354                         if (diag != 0)
2355                                 fprintf(stderr,
2356                                         "launch lcore %u failed - diag=%d\n",
2357                                         lc_id, diag);
2358                 }
2359         }
2360 }
2361
2362 /*
2363  * Launch packet forwarding configuration.
2364  */
2365 void
2366 start_packet_forwarding(int with_tx_first)
2367 {
2368         port_fwd_begin_t port_fwd_begin;
2369         port_fwd_end_t  port_fwd_end;
2370         stream_init_t stream_init = cur_fwd_eng->stream_init;
2371         unsigned int i;
2372
2373         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2374                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2375
2376         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2377                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2378
2379         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2380                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2381                 (!nb_rxq || !nb_txq))
2382                 rte_exit(EXIT_FAILURE,
2383                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2384                         cur_fwd_eng->fwd_mode_name);
2385
2386         if (all_ports_started() == 0) {
2387                 fprintf(stderr, "Not all ports were started\n");
2388                 return;
2389         }
2390         if (test_done == 0) {
2391                 fprintf(stderr, "Packet forwarding already started\n");
2392                 return;
2393         }
2394
2395         fwd_config_setup();
2396
2397         pkt_fwd_config_display(&cur_fwd_config);
2398         if (!pkt_fwd_shared_rxq_check())
2399                 return;
2400
2401         if (stream_init != NULL)
2402                 for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++)
2403                         stream_init(fwd_streams[i]);
2404
2405         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2406         if (port_fwd_begin != NULL) {
2407                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2408                         if (port_fwd_begin(fwd_ports_ids[i])) {
2409                                 fprintf(stderr,
2410                                         "Packet forwarding is not ready\n");
2411                                 return;
2412                         }
2413                 }
2414         }
2415
2416         if (with_tx_first) {
2417                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2418                 if (port_fwd_begin != NULL) {
2419                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2420                                 if (port_fwd_begin(fwd_ports_ids[i])) {
2421                                         fprintf(stderr,
2422                                                 "Packet forwarding is not ready\n");
2423                                         return;
2424                                 }
2425                         }
2426                 }
2427         }
2428
2429         test_done = 0;
2430
2431         if(!no_flush_rx)
2432                 flush_fwd_rx_queues();
2433
2434         rxtx_config_display();
2435
2436         fwd_stats_reset();
2437         if (with_tx_first) {
2438                 while (with_tx_first--) {
2439                         launch_packet_forwarding(
2440                                         run_one_txonly_burst_on_core);
2441                         rte_eal_mp_wait_lcore();
2442                 }
2443                 port_fwd_end = tx_only_engine.port_fwd_end;
2444                 if (port_fwd_end != NULL) {
2445                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2446                                 (*port_fwd_end)(fwd_ports_ids[i]);
2447                 }
2448         }
2449         launch_packet_forwarding(start_pkt_forward_on_core);
2450 }
2451
2452 void
2453 stop_packet_forwarding(void)
2454 {
2455         port_fwd_end_t port_fwd_end;
2456         lcoreid_t lc_id;
2457         portid_t pt_id;
2458         int i;
2459
2460         if (test_done) {
2461                 fprintf(stderr, "Packet forwarding not started\n");
2462                 return;
2463         }
2464         printf("Telling cores to stop...");
2465         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2466                 fwd_lcores[lc_id]->stopped = 1;
2467         printf("\nWaiting for lcores to finish...\n");
2468         rte_eal_mp_wait_lcore();
2469         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2470         if (port_fwd_end != NULL) {
2471                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2472                         pt_id = fwd_ports_ids[i];
2473                         (*port_fwd_end)(pt_id);
2474                 }
2475         }
2476
2477         fwd_stats_display();
2478
2479         printf("\nDone.\n");
2480         test_done = 1;
2481 }
2482
2483 void
2484 dev_set_link_up(portid_t pid)
2485 {
2486         if (rte_eth_dev_set_link_up(pid) < 0)
2487                 fprintf(stderr, "\nSet link up fail.\n");
2488 }
2489
2490 void
2491 dev_set_link_down(portid_t pid)
2492 {
2493         if (rte_eth_dev_set_link_down(pid) < 0)
2494                 fprintf(stderr, "\nSet link down fail.\n");
2495 }
2496
2497 static int
2498 all_ports_started(void)
2499 {
2500         portid_t pi;
2501         struct rte_port *port;
2502
2503         RTE_ETH_FOREACH_DEV(pi) {
2504                 port = &ports[pi];
2505                 /* Check if there is a port which is not started */
2506                 if ((port->port_status != RTE_PORT_STARTED) &&
2507                         (port->slave_flag == 0))
2508                         return 0;
2509         }
2510
2511         /* No port is not started */
2512         return 1;
2513 }
2514
2515 int
2516 port_is_stopped(portid_t port_id)
2517 {
2518         struct rte_port *port = &ports[port_id];
2519
2520         if ((port->port_status != RTE_PORT_STOPPED) &&
2521             (port->slave_flag == 0))
2522                 return 0;
2523         return 1;
2524 }
2525
2526 int
2527 all_ports_stopped(void)
2528 {
2529         portid_t pi;
2530
2531         RTE_ETH_FOREACH_DEV(pi) {
2532                 if (!port_is_stopped(pi))
2533                         return 0;
2534         }
2535
2536         return 1;
2537 }
2538
2539 int
2540 port_is_started(portid_t port_id)
2541 {
2542         if (port_id_is_invalid(port_id, ENABLED_WARN))
2543                 return 0;
2544
2545         if (ports[port_id].port_status != RTE_PORT_STARTED)
2546                 return 0;
2547
2548         return 1;
2549 }
2550
2551 /* Configure the Rx and Tx hairpin queues for the selected port. */
2552 static int
2553 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2554 {
2555         queueid_t qi;
2556         struct rte_eth_hairpin_conf hairpin_conf = {
2557                 .peer_count = 1,
2558         };
2559         int i;
2560         int diag;
2561         struct rte_port *port = &ports[pi];
2562         uint16_t peer_rx_port = pi;
2563         uint16_t peer_tx_port = pi;
2564         uint32_t manual = 1;
2565         uint32_t tx_exp = hairpin_mode & 0x10;
2566
2567         if (!(hairpin_mode & 0xf)) {
2568                 peer_rx_port = pi;
2569                 peer_tx_port = pi;
2570                 manual = 0;
2571         } else if (hairpin_mode & 0x1) {
2572                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2573                                                        RTE_ETH_DEV_NO_OWNER);
2574                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2575                         peer_tx_port = rte_eth_find_next_owned_by(0,
2576                                                 RTE_ETH_DEV_NO_OWNER);
2577                 if (p_pi != RTE_MAX_ETHPORTS) {
2578                         peer_rx_port = p_pi;
2579                 } else {
2580                         uint16_t next_pi;
2581
2582                         /* Last port will be the peer RX port of the first. */
2583                         RTE_ETH_FOREACH_DEV(next_pi)
2584                                 peer_rx_port = next_pi;
2585                 }
2586                 manual = 1;
2587         } else if (hairpin_mode & 0x2) {
2588                 if (cnt_pi & 0x1) {
2589                         peer_rx_port = p_pi;
2590                 } else {
2591                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2592                                                 RTE_ETH_DEV_NO_OWNER);
2593                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2594                                 peer_rx_port = pi;
2595                 }
2596                 peer_tx_port = peer_rx_port;
2597                 manual = 1;
2598         }
2599
2600         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2601                 hairpin_conf.peers[0].port = peer_rx_port;
2602                 hairpin_conf.peers[0].queue = i + nb_rxq;
2603                 hairpin_conf.manual_bind = !!manual;
2604                 hairpin_conf.tx_explicit = !!tx_exp;
2605                 diag = rte_eth_tx_hairpin_queue_setup
2606                         (pi, qi, nb_txd, &hairpin_conf);
2607                 i++;
2608                 if (diag == 0)
2609                         continue;
2610
2611                 /* Fail to setup rx queue, return */
2612                 if (port->port_status == RTE_PORT_HANDLING)
2613                         port->port_status = RTE_PORT_STOPPED;
2614                 else
2615                         fprintf(stderr,
2616                                 "Port %d can not be set back to stopped\n", pi);
2617                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2618                         pi);
2619                 /* try to reconfigure queues next time */
2620                 port->need_reconfig_queues = 1;
2621                 return -1;
2622         }
2623         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2624                 hairpin_conf.peers[0].port = peer_tx_port;
2625                 hairpin_conf.peers[0].queue = i + nb_txq;
2626                 hairpin_conf.manual_bind = !!manual;
2627                 hairpin_conf.tx_explicit = !!tx_exp;
2628                 diag = rte_eth_rx_hairpin_queue_setup
2629                         (pi, qi, nb_rxd, &hairpin_conf);
2630                 i++;
2631                 if (diag == 0)
2632                         continue;
2633
2634                 /* Fail to setup rx queue, return */
2635                 if (port->port_status == RTE_PORT_HANDLING)
2636                         port->port_status = RTE_PORT_STOPPED;
2637                 else
2638                         fprintf(stderr,
2639                                 "Port %d can not be set back to stopped\n", pi);
2640                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2641                         pi);
2642                 /* try to reconfigure queues next time */
2643                 port->need_reconfig_queues = 1;
2644                 return -1;
2645         }
2646         return 0;
2647 }
2648
2649 /* Configure the Rx with optional split. */
2650 int
2651 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2652                uint16_t nb_rx_desc, unsigned int socket_id,
2653                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2654 {
2655         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2656         unsigned int i, mp_n;
2657         int ret;
2658
2659         if (rx_pkt_nb_segs <= 1 ||
2660             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2661                 rx_conf->rx_seg = NULL;
2662                 rx_conf->rx_nseg = 0;
2663                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2664                                              nb_rx_desc, socket_id,
2665                                              rx_conf, mp);
2666                 goto exit;
2667         }
2668         for (i = 0; i < rx_pkt_nb_segs; i++) {
2669                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2670                 struct rte_mempool *mpx;
2671                 /*
2672                  * Use last valid pool for the segments with number
2673                  * exceeding the pool index.
2674                  */
2675                 mp_n = (i >= mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2676                 mpx = mbuf_pool_find(socket_id, mp_n);
2677                 /* Handle zero as mbuf data buffer size. */
2678                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2679                                    rx_pkt_seg_lengths[i] :
2680                                    mbuf_data_size[mp_n];
2681                 rx_seg->offset = i < rx_pkt_nb_offs ?
2682                                    rx_pkt_seg_offsets[i] : 0;
2683                 rx_seg->mp = mpx ? mpx : mp;
2684         }
2685         rx_conf->rx_nseg = rx_pkt_nb_segs;
2686         rx_conf->rx_seg = rx_useg;
2687         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2688                                     socket_id, rx_conf, NULL);
2689         rx_conf->rx_seg = NULL;
2690         rx_conf->rx_nseg = 0;
2691 exit:
2692         ports[port_id].rxq[rx_queue_id].state = rx_conf->rx_deferred_start ?
2693                                                 RTE_ETH_QUEUE_STATE_STOPPED :
2694                                                 RTE_ETH_QUEUE_STATE_STARTED;
2695         return ret;
2696 }
2697
2698 static int
2699 alloc_xstats_display_info(portid_t pi)
2700 {
2701         uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2702         uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2703         uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2704
2705         if (xstats_display_num == 0)
2706                 return 0;
2707
2708         *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2709         if (*ids_supp == NULL)
2710                 goto fail_ids_supp;
2711
2712         *prev_values = calloc(xstats_display_num,
2713                               sizeof(**prev_values));
2714         if (*prev_values == NULL)
2715                 goto fail_prev_values;
2716
2717         *curr_values = calloc(xstats_display_num,
2718                               sizeof(**curr_values));
2719         if (*curr_values == NULL)
2720                 goto fail_curr_values;
2721
2722         ports[pi].xstats_info.allocated = true;
2723
2724         return 0;
2725
2726 fail_curr_values:
2727         free(*prev_values);
2728 fail_prev_values:
2729         free(*ids_supp);
2730 fail_ids_supp:
2731         return -ENOMEM;
2732 }
2733
2734 static void
2735 free_xstats_display_info(portid_t pi)
2736 {
2737         if (!ports[pi].xstats_info.allocated)
2738                 return;
2739         free(ports[pi].xstats_info.ids_supp);
2740         free(ports[pi].xstats_info.prev_values);
2741         free(ports[pi].xstats_info.curr_values);
2742         ports[pi].xstats_info.allocated = false;
2743 }
2744
2745 /** Fill helper structures for specified port to show extended statistics. */
2746 static void
2747 fill_xstats_display_info_for_port(portid_t pi)
2748 {
2749         unsigned int stat, stat_supp;
2750         const char *xstat_name;
2751         struct rte_port *port;
2752         uint64_t *ids_supp;
2753         int rc;
2754
2755         if (xstats_display_num == 0)
2756                 return;
2757
2758         if (pi == (portid_t)RTE_PORT_ALL) {
2759                 fill_xstats_display_info();
2760                 return;
2761         }
2762
2763         port = &ports[pi];
2764         if (port->port_status != RTE_PORT_STARTED)
2765                 return;
2766
2767         if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2768                 rte_exit(EXIT_FAILURE,
2769                          "Failed to allocate xstats display memory\n");
2770
2771         ids_supp = port->xstats_info.ids_supp;
2772         for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2773                 xstat_name = xstats_display[stat].name;
2774                 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2775                                                    ids_supp + stat_supp);
2776                 if (rc != 0) {
2777                         fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2778                                 xstat_name, pi, stat);
2779                         continue;
2780                 }
2781                 stat_supp++;
2782         }
2783
2784         port->xstats_info.ids_supp_sz = stat_supp;
2785 }
2786
2787 /** Fill helper structures for all ports to show extended statistics. */
2788 static void
2789 fill_xstats_display_info(void)
2790 {
2791         portid_t pi;
2792
2793         if (xstats_display_num == 0)
2794                 return;
2795
2796         RTE_ETH_FOREACH_DEV(pi)
2797                 fill_xstats_display_info_for_port(pi);
2798 }
2799
2800 int
2801 start_port(portid_t pid)
2802 {
2803         int diag, need_check_link_status = -1;
2804         portid_t pi;
2805         portid_t p_pi = RTE_MAX_ETHPORTS;
2806         portid_t pl[RTE_MAX_ETHPORTS];
2807         portid_t peer_pl[RTE_MAX_ETHPORTS];
2808         uint16_t cnt_pi = 0;
2809         uint16_t cfg_pi = 0;
2810         int peer_pi;
2811         queueid_t qi;
2812         struct rte_port *port;
2813         struct rte_eth_hairpin_cap cap;
2814
2815         if (port_id_is_invalid(pid, ENABLED_WARN))
2816                 return 0;
2817
2818         RTE_ETH_FOREACH_DEV(pi) {
2819                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2820                         continue;
2821
2822                 if (port_is_bonding_slave(pi)) {
2823                         fprintf(stderr,
2824                                 "Please remove port %d from bonded device.\n",
2825                                 pi);
2826                         continue;
2827                 }
2828
2829                 need_check_link_status = 0;
2830                 port = &ports[pi];
2831                 if (port->port_status == RTE_PORT_STOPPED)
2832                         port->port_status = RTE_PORT_HANDLING;
2833                 else {
2834                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2835                         continue;
2836                 }
2837
2838                 if (port->need_reconfig > 0) {
2839                         struct rte_eth_conf dev_conf;
2840                         int k;
2841
2842                         port->need_reconfig = 0;
2843
2844                         if (flow_isolate_all) {
2845                                 int ret = port_flow_isolate(pi, 1);
2846                                 if (ret) {
2847                                         fprintf(stderr,
2848                                                 "Failed to apply isolated mode on port %d\n",
2849                                                 pi);
2850                                         return -1;
2851                                 }
2852                         }
2853                         configure_rxtx_dump_callbacks(0);
2854                         printf("Configuring Port %d (socket %u)\n", pi,
2855                                         port->socket_id);
2856                         if (nb_hairpinq > 0 &&
2857                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2858                                 fprintf(stderr,
2859                                         "Port %d doesn't support hairpin queues\n",
2860                                         pi);
2861                                 return -1;
2862                         }
2863
2864                         /* configure port */
2865                         diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2866                                                      nb_txq + nb_hairpinq,
2867                                                      &(port->dev_conf));
2868                         if (diag != 0) {
2869                                 if (port->port_status == RTE_PORT_HANDLING)
2870                                         port->port_status = RTE_PORT_STOPPED;
2871                                 else
2872                                         fprintf(stderr,
2873                                                 "Port %d can not be set back to stopped\n",
2874                                                 pi);
2875                                 fprintf(stderr, "Fail to configure port %d\n",
2876                                         pi);
2877                                 /* try to reconfigure port next time */
2878                                 port->need_reconfig = 1;
2879                                 return -1;
2880                         }
2881                         /* get device configuration*/
2882                         if (0 !=
2883                                 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2884                                 fprintf(stderr,
2885                                         "port %d can not get device configuration\n",
2886                                         pi);
2887                                 return -1;
2888                         }
2889                         /* Apply Rx offloads configuration */
2890                         if (dev_conf.rxmode.offloads !=
2891                             port->dev_conf.rxmode.offloads) {
2892                                 port->dev_conf.rxmode.offloads |=
2893                                         dev_conf.rxmode.offloads;
2894                                 for (k = 0;
2895                                      k < port->dev_info.max_rx_queues;
2896                                      k++)
2897                                         port->rxq[k].conf.offloads |=
2898                                                 dev_conf.rxmode.offloads;
2899                         }
2900                         /* Apply Tx offloads configuration */
2901                         if (dev_conf.txmode.offloads !=
2902                             port->dev_conf.txmode.offloads) {
2903                                 port->dev_conf.txmode.offloads |=
2904                                         dev_conf.txmode.offloads;
2905                                 for (k = 0;
2906                                      k < port->dev_info.max_tx_queues;
2907                                      k++)
2908                                         port->txq[k].conf.offloads |=
2909                                                 dev_conf.txmode.offloads;
2910                         }
2911                 }
2912                 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2913                         port->need_reconfig_queues = 0;
2914                         /* setup tx queues */
2915                         for (qi = 0; qi < nb_txq; qi++) {
2916                                 struct rte_eth_txconf *conf =
2917                                                         &port->txq[qi].conf;
2918
2919                                 if ((numa_support) &&
2920                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2921                                         diag = rte_eth_tx_queue_setup(pi, qi,
2922                                                 port->nb_tx_desc[qi],
2923                                                 txring_numa[pi],
2924                                                 &(port->txq[qi].conf));
2925                                 else
2926                                         diag = rte_eth_tx_queue_setup(pi, qi,
2927                                                 port->nb_tx_desc[qi],
2928                                                 port->socket_id,
2929                                                 &(port->txq[qi].conf));
2930
2931                                 if (diag == 0) {
2932                                         port->txq[qi].state =
2933                                                 conf->tx_deferred_start ?
2934                                                 RTE_ETH_QUEUE_STATE_STOPPED :
2935                                                 RTE_ETH_QUEUE_STATE_STARTED;
2936                                         continue;
2937                                 }
2938
2939                                 /* Fail to setup tx queue, return */
2940                                 if (port->port_status == RTE_PORT_HANDLING)
2941                                         port->port_status = RTE_PORT_STOPPED;
2942                                 else
2943                                         fprintf(stderr,
2944                                                 "Port %d can not be set back to stopped\n",
2945                                                 pi);
2946                                 fprintf(stderr,
2947                                         "Fail to configure port %d tx queues\n",
2948                                         pi);
2949                                 /* try to reconfigure queues next time */
2950                                 port->need_reconfig_queues = 1;
2951                                 return -1;
2952                         }
2953                         for (qi = 0; qi < nb_rxq; qi++) {
2954                                 /* setup rx queues */
2955                                 if ((numa_support) &&
2956                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2957                                         struct rte_mempool * mp =
2958                                                 mbuf_pool_find
2959                                                         (rxring_numa[pi], 0);
2960                                         if (mp == NULL) {
2961                                                 fprintf(stderr,
2962                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2963                                                         rxring_numa[pi]);
2964                                                 return -1;
2965                                         }
2966
2967                                         diag = rx_queue_setup(pi, qi,
2968                                              port->nb_rx_desc[qi],
2969                                              rxring_numa[pi],
2970                                              &(port->rxq[qi].conf),
2971                                              mp);
2972                                 } else {
2973                                         struct rte_mempool *mp =
2974                                                 mbuf_pool_find
2975                                                         (port->socket_id, 0);
2976                                         if (mp == NULL) {
2977                                                 fprintf(stderr,
2978                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2979                                                         port->socket_id);
2980                                                 return -1;
2981                                         }
2982                                         diag = rx_queue_setup(pi, qi,
2983                                              port->nb_rx_desc[qi],
2984                                              port->socket_id,
2985                                              &(port->rxq[qi].conf),
2986                                              mp);
2987                                 }
2988                                 if (diag == 0)
2989                                         continue;
2990
2991                                 /* Fail to setup rx queue, return */
2992                                 if (port->port_status == RTE_PORT_HANDLING)
2993                                         port->port_status = RTE_PORT_STOPPED;
2994                                 else
2995                                         fprintf(stderr,
2996                                                 "Port %d can not be set back to stopped\n",
2997                                                 pi);
2998                                 fprintf(stderr,
2999                                         "Fail to configure port %d rx queues\n",
3000                                         pi);
3001                                 /* try to reconfigure queues next time */
3002                                 port->need_reconfig_queues = 1;
3003                                 return -1;
3004                         }
3005                         /* setup hairpin queues */
3006                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
3007                                 return -1;
3008                 }
3009                 configure_rxtx_dump_callbacks(verbose_level);
3010                 if (clear_ptypes) {
3011                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
3012                                         NULL, 0);
3013                         if (diag < 0)
3014                                 fprintf(stderr,
3015                                         "Port %d: Failed to disable Ptype parsing\n",
3016                                         pi);
3017                 }
3018
3019                 p_pi = pi;
3020                 cnt_pi++;
3021
3022                 /* start port */
3023                 diag = eth_dev_start_mp(pi);
3024                 if (diag < 0) {
3025                         fprintf(stderr, "Fail to start port %d: %s\n",
3026                                 pi, rte_strerror(-diag));
3027
3028                         /* Fail to setup rx queue, return */
3029                         if (port->port_status == RTE_PORT_HANDLING)
3030                                 port->port_status = RTE_PORT_STOPPED;
3031                         else
3032                                 fprintf(stderr,
3033                                         "Port %d can not be set back to stopped\n",
3034                                         pi);
3035                         continue;
3036                 }
3037
3038                 if (port->port_status == RTE_PORT_HANDLING)
3039                         port->port_status = RTE_PORT_STARTED;
3040                 else
3041                         fprintf(stderr, "Port %d can not be set into started\n",
3042                                 pi);
3043
3044                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
3045                         printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
3046                                         RTE_ETHER_ADDR_BYTES(&port->eth_addr));
3047
3048                 /* at least one port started, need checking link status */
3049                 need_check_link_status = 1;
3050
3051                 pl[cfg_pi++] = pi;
3052         }
3053
3054         if (need_check_link_status == 1 && !no_link_check)
3055                 check_all_ports_link_status(RTE_PORT_ALL);
3056         else if (need_check_link_status == 0)
3057                 fprintf(stderr, "Please stop the ports first\n");
3058
3059         if (hairpin_mode & 0xf) {
3060                 uint16_t i;
3061                 int j;
3062
3063                 /* bind all started hairpin ports */
3064                 for (i = 0; i < cfg_pi; i++) {
3065                         pi = pl[i];
3066                         /* bind current Tx to all peer Rx */
3067                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3068                                                         RTE_MAX_ETHPORTS, 1);
3069                         if (peer_pi < 0)
3070                                 return peer_pi;
3071                         for (j = 0; j < peer_pi; j++) {
3072                                 if (!port_is_started(peer_pl[j]))
3073                                         continue;
3074                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
3075                                 if (diag < 0) {
3076                                         fprintf(stderr,
3077                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3078                                                 pi, peer_pl[j],
3079                                                 rte_strerror(-diag));
3080                                         return -1;
3081                                 }
3082                         }
3083                         /* bind all peer Tx to current Rx */
3084                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3085                                                         RTE_MAX_ETHPORTS, 0);
3086                         if (peer_pi < 0)
3087                                 return peer_pi;
3088                         for (j = 0; j < peer_pi; j++) {
3089                                 if (!port_is_started(peer_pl[j]))
3090                                         continue;
3091                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
3092                                 if (diag < 0) {
3093                                         fprintf(stderr,
3094                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
3095                                                 peer_pl[j], pi,
3096                                                 rte_strerror(-diag));
3097                                         return -1;
3098                                 }
3099                         }
3100                 }
3101         }
3102
3103         fill_xstats_display_info_for_port(pid);
3104
3105         printf("Done\n");
3106         return 0;
3107 }
3108
3109 void
3110 stop_port(portid_t pid)
3111 {
3112         portid_t pi;
3113         struct rte_port *port;
3114         int need_check_link_status = 0;
3115         portid_t peer_pl[RTE_MAX_ETHPORTS];
3116         int peer_pi;
3117
3118         if (port_id_is_invalid(pid, ENABLED_WARN))
3119                 return;
3120
3121         printf("Stopping ports...\n");
3122
3123         RTE_ETH_FOREACH_DEV(pi) {
3124                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3125                         continue;
3126
3127                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3128                         fprintf(stderr,
3129                                 "Please remove port %d from forwarding configuration.\n",
3130                                 pi);
3131                         continue;
3132                 }
3133
3134                 if (port_is_bonding_slave(pi)) {
3135                         fprintf(stderr,
3136                                 "Please remove port %d from bonded device.\n",
3137                                 pi);
3138                         continue;
3139                 }
3140
3141                 port = &ports[pi];
3142                 if (port->port_status == RTE_PORT_STARTED)
3143                         port->port_status = RTE_PORT_HANDLING;
3144                 else
3145                         continue;
3146
3147                 if (hairpin_mode & 0xf) {
3148                         int j;
3149
3150                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3151                         /* unbind all peer Tx from current Rx */
3152                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3153                                                         RTE_MAX_ETHPORTS, 0);
3154                         if (peer_pi < 0)
3155                                 continue;
3156                         for (j = 0; j < peer_pi; j++) {
3157                                 if (!port_is_started(peer_pl[j]))
3158                                         continue;
3159                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
3160                         }
3161                 }
3162
3163                 if (port->flow_list)
3164                         port_flow_flush(pi);
3165
3166                 if (eth_dev_stop_mp(pi) != 0)
3167                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3168                                 pi);
3169
3170                 if (port->port_status == RTE_PORT_HANDLING)
3171                         port->port_status = RTE_PORT_STOPPED;
3172                 else
3173                         fprintf(stderr, "Port %d can not be set into stopped\n",
3174                                 pi);
3175                 need_check_link_status = 1;
3176         }
3177         if (need_check_link_status && !no_link_check)
3178                 check_all_ports_link_status(RTE_PORT_ALL);
3179
3180         printf("Done\n");
3181 }
3182
3183 static void
3184 remove_invalid_ports_in(portid_t *array, portid_t *total)
3185 {
3186         portid_t i;
3187         portid_t new_total = 0;
3188
3189         for (i = 0; i < *total; i++)
3190                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3191                         array[new_total] = array[i];
3192                         new_total++;
3193                 }
3194         *total = new_total;
3195 }
3196
3197 static void
3198 remove_invalid_ports(void)
3199 {
3200         remove_invalid_ports_in(ports_ids, &nb_ports);
3201         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3202         nb_cfg_ports = nb_fwd_ports;
3203 }
3204
3205 void
3206 close_port(portid_t pid)
3207 {
3208         portid_t pi;
3209         struct rte_port *port;
3210
3211         if (port_id_is_invalid(pid, ENABLED_WARN))
3212                 return;
3213
3214         printf("Closing ports...\n");
3215
3216         RTE_ETH_FOREACH_DEV(pi) {
3217                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3218                         continue;
3219
3220                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3221                         fprintf(stderr,
3222                                 "Please remove port %d from forwarding configuration.\n",
3223                                 pi);
3224                         continue;
3225                 }
3226
3227                 if (port_is_bonding_slave(pi)) {
3228                         fprintf(stderr,
3229                                 "Please remove port %d from bonded device.\n",
3230                                 pi);
3231                         continue;
3232                 }
3233
3234                 port = &ports[pi];
3235                 if (port->port_status == RTE_PORT_CLOSED) {
3236                         fprintf(stderr, "Port %d is already closed\n", pi);
3237                         continue;
3238                 }
3239
3240                 if (is_proc_primary()) {
3241                         mcast_addr_pool_destroy(pi);
3242                         port_flow_flush(pi);
3243                         port_flex_item_flush(pi);
3244                         port_action_handle_flush(pi);
3245                         rte_eth_dev_close(pi);
3246                 }
3247
3248                 free_xstats_display_info(pi);
3249         }
3250
3251         remove_invalid_ports();
3252         printf("Done\n");
3253 }
3254
3255 void
3256 reset_port(portid_t pid)
3257 {
3258         int diag;
3259         portid_t pi;
3260         struct rte_port *port;
3261
3262         if (port_id_is_invalid(pid, ENABLED_WARN))
3263                 return;
3264
3265         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3266                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3267                 fprintf(stderr,
3268                         "Can not reset port(s), please stop port(s) first.\n");
3269                 return;
3270         }
3271
3272         printf("Resetting ports...\n");
3273
3274         RTE_ETH_FOREACH_DEV(pi) {
3275                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3276                         continue;
3277
3278                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3279                         fprintf(stderr,
3280                                 "Please remove port %d from forwarding configuration.\n",
3281                                 pi);
3282                         continue;
3283                 }
3284
3285                 if (port_is_bonding_slave(pi)) {
3286                         fprintf(stderr,
3287                                 "Please remove port %d from bonded device.\n",
3288                                 pi);
3289                         continue;
3290                 }
3291
3292                 diag = rte_eth_dev_reset(pi);
3293                 if (diag == 0) {
3294                         port = &ports[pi];
3295                         port->need_reconfig = 1;
3296                         port->need_reconfig_queues = 1;
3297                 } else {
3298                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3299                                 pi, diag);
3300                 }
3301         }
3302
3303         printf("Done\n");
3304 }
3305
3306 void
3307 attach_port(char *identifier)
3308 {
3309         portid_t pi;
3310         struct rte_dev_iterator iterator;
3311
3312         printf("Attaching a new port...\n");
3313
3314         if (identifier == NULL) {
3315                 fprintf(stderr, "Invalid parameters are specified\n");
3316                 return;
3317         }
3318
3319         if (rte_dev_probe(identifier) < 0) {
3320                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3321                 return;
3322         }
3323
3324         /* first attach mode: event */
3325         if (setup_on_probe_event) {
3326                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3327                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3328                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
3329                                         ports[pi].need_setup != 0)
3330                                 setup_attached_port(pi);
3331                 return;
3332         }
3333
3334         /* second attach mode: iterator */
3335         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3336                 /* setup ports matching the devargs used for probing */
3337                 if (port_is_forwarding(pi))
3338                         continue; /* port was already attached before */
3339                 setup_attached_port(pi);
3340         }
3341 }
3342
3343 static void
3344 setup_attached_port(portid_t pi)
3345 {
3346         unsigned int socket_id;
3347         int ret;
3348
3349         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3350         /* if socket_id is invalid, set to the first available socket. */
3351         if (check_socket_id(socket_id) < 0)
3352                 socket_id = socket_ids[0];
3353         reconfig(pi, socket_id);
3354         ret = rte_eth_promiscuous_enable(pi);
3355         if (ret != 0)
3356                 fprintf(stderr,
3357                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3358                         pi, rte_strerror(-ret));
3359
3360         ports_ids[nb_ports++] = pi;
3361         fwd_ports_ids[nb_fwd_ports++] = pi;
3362         nb_cfg_ports = nb_fwd_ports;
3363         ports[pi].need_setup = 0;
3364         ports[pi].port_status = RTE_PORT_STOPPED;
3365
3366         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3367         printf("Done\n");
3368 }
3369
3370 static void
3371 detach_device(struct rte_device *dev)
3372 {
3373         portid_t sibling;
3374
3375         if (dev == NULL) {
3376                 fprintf(stderr, "Device already removed\n");
3377                 return;
3378         }
3379
3380         printf("Removing a device...\n");
3381
3382         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3383                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3384                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3385                                 fprintf(stderr, "Port %u not stopped\n",
3386                                         sibling);
3387                                 return;
3388                         }
3389                         port_flow_flush(sibling);
3390                 }
3391         }
3392
3393         if (rte_dev_remove(dev) < 0) {
3394                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3395                 return;
3396         }
3397         remove_invalid_ports();
3398
3399         printf("Device is detached\n");
3400         printf("Now total ports is %d\n", nb_ports);
3401         printf("Done\n");
3402         return;
3403 }
3404
3405 void
3406 detach_port_device(portid_t port_id)
3407 {
3408         int ret;
3409         struct rte_eth_dev_info dev_info;
3410
3411         if (port_id_is_invalid(port_id, ENABLED_WARN))
3412                 return;
3413
3414         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3415                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3416                         fprintf(stderr, "Port not stopped\n");
3417                         return;
3418                 }
3419                 fprintf(stderr, "Port was not closed\n");
3420         }
3421
3422         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3423         if (ret != 0) {
3424                 TESTPMD_LOG(ERR,
3425                         "Failed to get device info for port %d, not detaching\n",
3426                         port_id);
3427                 return;
3428         }
3429         detach_device(dev_info.device);
3430 }
3431
3432 void
3433 detach_devargs(char *identifier)
3434 {
3435         struct rte_dev_iterator iterator;
3436         struct rte_devargs da;
3437         portid_t port_id;
3438
3439         printf("Removing a device...\n");
3440
3441         memset(&da, 0, sizeof(da));
3442         if (rte_devargs_parsef(&da, "%s", identifier)) {
3443                 fprintf(stderr, "cannot parse identifier\n");
3444                 return;
3445         }
3446
3447         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3448                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3449                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3450                                 fprintf(stderr, "Port %u not stopped\n",
3451                                         port_id);
3452                                 rte_eth_iterator_cleanup(&iterator);
3453                                 rte_devargs_reset(&da);
3454                                 return;
3455                         }
3456                         port_flow_flush(port_id);
3457                 }
3458         }
3459
3460         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3461                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3462                             da.name, da.bus->name);
3463                 rte_devargs_reset(&da);
3464                 return;
3465         }
3466
3467         remove_invalid_ports();
3468
3469         printf("Device %s is detached\n", identifier);
3470         printf("Now total ports is %d\n", nb_ports);
3471         printf("Done\n");
3472         rte_devargs_reset(&da);
3473 }
3474
3475 void
3476 pmd_test_exit(void)
3477 {
3478         portid_t pt_id;
3479         unsigned int i;
3480         int ret;
3481
3482         if (test_done == 0)
3483                 stop_packet_forwarding();
3484
3485 #ifndef RTE_EXEC_ENV_WINDOWS
3486         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3487                 if (mempools[i]) {
3488                         if (mp_alloc_type == MP_ALLOC_ANON)
3489                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3490                                                      NULL);
3491                 }
3492         }
3493 #endif
3494         if (ports != NULL) {
3495                 no_link_check = 1;
3496                 RTE_ETH_FOREACH_DEV(pt_id) {
3497                         printf("\nStopping port %d...\n", pt_id);
3498                         fflush(stdout);
3499                         stop_port(pt_id);
3500                 }
3501                 RTE_ETH_FOREACH_DEV(pt_id) {
3502                         printf("\nShutting down port %d...\n", pt_id);
3503                         fflush(stdout);
3504                         close_port(pt_id);
3505                 }
3506         }
3507
3508         if (hot_plug) {
3509                 ret = rte_dev_event_monitor_stop();
3510                 if (ret) {
3511                         RTE_LOG(ERR, EAL,
3512                                 "fail to stop device event monitor.");
3513                         return;
3514                 }
3515
3516                 ret = rte_dev_event_callback_unregister(NULL,
3517                         dev_event_callback, NULL);
3518                 if (ret < 0) {
3519                         RTE_LOG(ERR, EAL,
3520                                 "fail to unregister device event callback.\n");
3521                         return;
3522                 }
3523
3524                 ret = rte_dev_hotplug_handle_disable();
3525                 if (ret) {
3526                         RTE_LOG(ERR, EAL,
3527                                 "fail to disable hotplug handling.\n");
3528                         return;
3529                 }
3530         }
3531         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3532                 if (mempools[i])
3533                         mempool_free_mp(mempools[i]);
3534         }
3535         free(xstats_display);
3536
3537         printf("\nBye...\n");
3538 }
3539
3540 typedef void (*cmd_func_t)(void);
3541 struct pmd_test_command {
3542         const char *cmd_name;
3543         cmd_func_t cmd_func;
3544 };
3545
3546 /* Check the link status of all ports in up to 9s, and print them finally */
3547 static void
3548 check_all_ports_link_status(uint32_t port_mask)
3549 {
3550 #define CHECK_INTERVAL 100 /* 100ms */
3551 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3552         portid_t portid;
3553         uint8_t count, all_ports_up, print_flag = 0;
3554         struct rte_eth_link link;
3555         int ret;
3556         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3557
3558         printf("Checking link statuses...\n");
3559         fflush(stdout);
3560         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3561                 all_ports_up = 1;
3562                 RTE_ETH_FOREACH_DEV(portid) {
3563                         if ((port_mask & (1 << portid)) == 0)
3564                                 continue;
3565                         memset(&link, 0, sizeof(link));
3566                         ret = rte_eth_link_get_nowait(portid, &link);
3567                         if (ret < 0) {
3568                                 all_ports_up = 0;
3569                                 if (print_flag == 1)
3570                                         fprintf(stderr,
3571                                                 "Port %u link get failed: %s\n",
3572                                                 portid, rte_strerror(-ret));
3573                                 continue;
3574                         }
3575                         /* print link status if flag set */
3576                         if (print_flag == 1) {
3577                                 rte_eth_link_to_str(link_status,
3578                                         sizeof(link_status), &link);
3579                                 printf("Port %d %s\n", portid, link_status);
3580                                 continue;
3581                         }
3582                         /* clear all_ports_up flag if any link down */
3583                         if (link.link_status == RTE_ETH_LINK_DOWN) {
3584                                 all_ports_up = 0;
3585                                 break;
3586                         }
3587                 }
3588                 /* after finally printing all link status, get out */
3589                 if (print_flag == 1)
3590                         break;
3591
3592                 if (all_ports_up == 0) {
3593                         fflush(stdout);
3594                         rte_delay_ms(CHECK_INTERVAL);
3595                 }
3596
3597                 /* set the print_flag if all ports up or timeout */
3598                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3599                         print_flag = 1;
3600                 }
3601
3602                 if (lsc_interrupt)
3603                         break;
3604         }
3605 }
3606
3607 static void
3608 rmv_port_callback(void *arg)
3609 {
3610         int need_to_start = 0;
3611         int org_no_link_check = no_link_check;
3612         portid_t port_id = (intptr_t)arg;
3613         struct rte_eth_dev_info dev_info;
3614         int ret;
3615
3616         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3617
3618         if (!test_done && port_is_forwarding(port_id)) {
3619                 need_to_start = 1;
3620                 stop_packet_forwarding();
3621         }
3622         no_link_check = 1;
3623         stop_port(port_id);
3624         no_link_check = org_no_link_check;
3625
3626         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3627         if (ret != 0)
3628                 TESTPMD_LOG(ERR,
3629                         "Failed to get device info for port %d, not detaching\n",
3630                         port_id);
3631         else {
3632                 struct rte_device *device = dev_info.device;
3633                 close_port(port_id);
3634                 detach_device(device); /* might be already removed or have more ports */
3635         }
3636         if (need_to_start)
3637                 start_packet_forwarding(0);
3638 }
3639
3640 /* This function is used by the interrupt thread */
3641 static int
3642 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3643                   void *ret_param)
3644 {
3645         RTE_SET_USED(param);
3646         RTE_SET_USED(ret_param);
3647
3648         if (type >= RTE_ETH_EVENT_MAX) {
3649                 fprintf(stderr,
3650                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3651                         port_id, __func__, type);
3652                 fflush(stderr);
3653         } else if (event_print_mask & (UINT32_C(1) << type)) {
3654                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3655                         eth_event_desc[type]);
3656                 fflush(stdout);
3657         }
3658
3659         switch (type) {
3660         case RTE_ETH_EVENT_NEW:
3661                 ports[port_id].need_setup = 1;
3662                 ports[port_id].port_status = RTE_PORT_HANDLING;
3663                 break;
3664         case RTE_ETH_EVENT_INTR_RMV:
3665                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3666                         break;
3667                 if (rte_eal_alarm_set(100000,
3668                                 rmv_port_callback, (void *)(intptr_t)port_id))
3669                         fprintf(stderr,
3670                                 "Could not set up deferred device removal\n");
3671                 break;
3672         case RTE_ETH_EVENT_DESTROY:
3673                 ports[port_id].port_status = RTE_PORT_CLOSED;
3674                 printf("Port %u is closed\n", port_id);
3675                 break;
3676         case RTE_ETH_EVENT_RX_AVAIL_THRESH: {
3677                 uint16_t rxq_id;
3678                 int ret;
3679
3680                 /* avail_thresh query API rewinds rxq_id, no need to check max RxQ num */
3681                 for (rxq_id = 0; ; rxq_id++) {
3682                         ret = rte_eth_rx_avail_thresh_query(port_id, &rxq_id,
3683                                                             NULL);
3684                         if (ret <= 0)
3685                                 break;
3686                         printf("Received avail_thresh event, port: %u, rxq_id: %u\n",
3687                                port_id, rxq_id);
3688                 }
3689                 break;
3690         }
3691         default:
3692                 break;
3693         }
3694         return 0;
3695 }
3696
3697 static int
3698 register_eth_event_callback(void)
3699 {
3700         int ret;
3701         enum rte_eth_event_type event;
3702
3703         for (event = RTE_ETH_EVENT_UNKNOWN;
3704                         event < RTE_ETH_EVENT_MAX; event++) {
3705                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3706                                 event,
3707                                 eth_event_callback,
3708                                 NULL);
3709                 if (ret != 0) {
3710                         TESTPMD_LOG(ERR, "Failed to register callback for "
3711                                         "%s event\n", eth_event_desc[event]);
3712                         return -1;
3713                 }
3714         }
3715
3716         return 0;
3717 }
3718
3719 /* This function is used by the interrupt thread */
3720 static void
3721 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3722                              __rte_unused void *arg)
3723 {
3724         uint16_t port_id;
3725         int ret;
3726
3727         if (type >= RTE_DEV_EVENT_MAX) {
3728                 fprintf(stderr, "%s called upon invalid event %d\n",
3729                         __func__, type);
3730                 fflush(stderr);
3731         }
3732
3733         switch (type) {
3734         case RTE_DEV_EVENT_REMOVE:
3735                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3736                         device_name);
3737                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3738                 if (ret) {
3739                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3740                                 device_name);
3741                         return;
3742                 }
3743                 /*
3744                  * Because the user's callback is invoked in eal interrupt
3745                  * callback, the interrupt callback need to be finished before
3746                  * it can be unregistered when detaching device. So finish
3747                  * callback soon and use a deferred removal to detach device
3748                  * is need. It is a workaround, once the device detaching be
3749                  * moved into the eal in the future, the deferred removal could
3750                  * be deleted.
3751                  */
3752                 if (rte_eal_alarm_set(100000,
3753                                 rmv_port_callback, (void *)(intptr_t)port_id))
3754                         RTE_LOG(ERR, EAL,
3755                                 "Could not set up deferred device removal\n");
3756                 break;
3757         case RTE_DEV_EVENT_ADD:
3758                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3759                         device_name);
3760                 /* TODO: After finish kernel driver binding,
3761                  * begin to attach port.
3762                  */
3763                 break;
3764         default:
3765                 break;
3766         }
3767 }
3768
3769 static void
3770 rxtx_port_config(portid_t pid)
3771 {
3772         uint16_t qid;
3773         uint64_t offloads;
3774         struct rte_port *port = &ports[pid];
3775
3776         for (qid = 0; qid < nb_rxq; qid++) {
3777                 offloads = port->rxq[qid].conf.offloads;
3778                 port->rxq[qid].conf = port->dev_info.default_rxconf;
3779
3780                 if (rxq_share > 0 &&
3781                     (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3782                         /* Non-zero share group to enable RxQ share. */
3783                         port->rxq[qid].conf.share_group = pid / rxq_share + 1;
3784                         port->rxq[qid].conf.share_qid = qid; /* Equal mapping. */
3785                 }
3786
3787                 if (offloads != 0)
3788                         port->rxq[qid].conf.offloads = offloads;
3789
3790                 /* Check if any Rx parameters have been passed */
3791                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3792                         port->rxq[qid].conf.rx_thresh.pthresh = rx_pthresh;
3793
3794                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3795                         port->rxq[qid].conf.rx_thresh.hthresh = rx_hthresh;
3796
3797                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3798                         port->rxq[qid].conf.rx_thresh.wthresh = rx_wthresh;
3799
3800                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3801                         port->rxq[qid].conf.rx_free_thresh = rx_free_thresh;
3802
3803                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3804                         port->rxq[qid].conf.rx_drop_en = rx_drop_en;
3805
3806                 port->nb_rx_desc[qid] = nb_rxd;
3807         }
3808
3809         for (qid = 0; qid < nb_txq; qid++) {
3810                 offloads = port->txq[qid].conf.offloads;
3811                 port->txq[qid].conf = port->dev_info.default_txconf;
3812                 if (offloads != 0)
3813                         port->txq[qid].conf.offloads = offloads;
3814
3815                 /* Check if any Tx parameters have been passed */
3816                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3817                         port->txq[qid].conf.tx_thresh.pthresh = tx_pthresh;
3818
3819                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3820                         port->txq[qid].conf.tx_thresh.hthresh = tx_hthresh;
3821
3822                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3823                         port->txq[qid].conf.tx_thresh.wthresh = tx_wthresh;
3824
3825                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3826                         port->txq[qid].conf.tx_rs_thresh = tx_rs_thresh;
3827
3828                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3829                         port->txq[qid].conf.tx_free_thresh = tx_free_thresh;
3830
3831                 port->nb_tx_desc[qid] = nb_txd;
3832         }
3833 }
3834
3835 /*
3836  * Helper function to set MTU from frame size
3837  *
3838  * port->dev_info should be set before calling this function.
3839  *
3840  * return 0 on success, negative on error
3841  */
3842 int
3843 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3844 {
3845         struct rte_port *port = &ports[portid];
3846         uint32_t eth_overhead;
3847         uint16_t mtu, new_mtu;
3848
3849         eth_overhead = get_eth_overhead(&port->dev_info);
3850
3851         if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3852                 printf("Failed to get MTU for port %u\n", portid);
3853                 return -1;
3854         }
3855
3856         new_mtu = max_rx_pktlen - eth_overhead;
3857
3858         if (mtu == new_mtu)
3859                 return 0;
3860
3861         if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3862                 fprintf(stderr,
3863                         "Failed to set MTU to %u for port %u\n",
3864                         new_mtu, portid);
3865                 return -1;
3866         }
3867
3868         port->dev_conf.rxmode.mtu = new_mtu;
3869
3870         return 0;
3871 }
3872
3873 void
3874 init_port_config(void)
3875 {
3876         portid_t pid;
3877         struct rte_port *port;
3878         int ret, i;
3879
3880         RTE_ETH_FOREACH_DEV(pid) {
3881                 port = &ports[pid];
3882                 port->dev_conf.fdir_conf = fdir_conf;
3883
3884                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3885                 if (ret != 0)
3886                         return;
3887
3888                 if (nb_rxq > 1) {
3889                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3890                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3891                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3892                 } else {
3893                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3894                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3895                 }
3896
3897                 if (port->dcb_flag == 0) {
3898                         if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3899                                 port->dev_conf.rxmode.mq_mode =
3900                                         (enum rte_eth_rx_mq_mode)
3901                                                 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3902                         } else {
3903                                 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3904                                 port->dev_conf.rxmode.offloads &=
3905                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3906
3907                                 for (i = 0;
3908                                      i < port->dev_info.nb_rx_queues;
3909                                      i++)
3910                                         port->rxq[i].conf.offloads &=
3911                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3912                         }
3913                 }
3914
3915                 rxtx_port_config(pid);
3916
3917                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3918                 if (ret != 0)
3919                         return;
3920
3921 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3922                 rte_pmd_ixgbe_bypass_init(pid);
3923 #endif
3924
3925                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3926                         port->dev_conf.intr_conf.lsc = 1;
3927                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3928                         port->dev_conf.intr_conf.rmv = 1;
3929         }
3930 }
3931
3932 void set_port_slave_flag(portid_t slave_pid)
3933 {
3934         struct rte_port *port;
3935
3936         port = &ports[slave_pid];
3937         port->slave_flag = 1;
3938 }
3939
3940 void clear_port_slave_flag(portid_t slave_pid)
3941 {
3942         struct rte_port *port;
3943
3944         port = &ports[slave_pid];
3945         port->slave_flag = 0;
3946 }
3947
3948 uint8_t port_is_bonding_slave(portid_t slave_pid)
3949 {
3950         struct rte_port *port;
3951         struct rte_eth_dev_info dev_info;
3952         int ret;
3953
3954         port = &ports[slave_pid];
3955         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3956         if (ret != 0) {
3957                 TESTPMD_LOG(ERR,
3958                         "Failed to get device info for port id %d,"
3959                         "cannot determine if the port is a bonded slave",
3960                         slave_pid);
3961                 return 0;
3962         }
3963         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3964                 return 1;
3965         return 0;
3966 }
3967
3968 const uint16_t vlan_tags[] = {
3969                 0,  1,  2,  3,  4,  5,  6,  7,
3970                 8,  9, 10, 11,  12, 13, 14, 15,
3971                 16, 17, 18, 19, 20, 21, 22, 23,
3972                 24, 25, 26, 27, 28, 29, 30, 31
3973 };
3974
3975 static  int
3976 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3977                  enum dcb_mode_enable dcb_mode,
3978                  enum rte_eth_nb_tcs num_tcs,
3979                  uint8_t pfc_en)
3980 {
3981         uint8_t i;
3982         int32_t rc;
3983         struct rte_eth_rss_conf rss_conf;
3984
3985         /*
3986          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3987          * given above, and the number of traffic classes available for use.
3988          */
3989         if (dcb_mode == DCB_VT_ENABLED) {
3990                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3991                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3992                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3993                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3994
3995                 /* VMDQ+DCB RX and TX configurations */
3996                 vmdq_rx_conf->enable_default_pool = 0;
3997                 vmdq_rx_conf->default_pool = 0;
3998                 vmdq_rx_conf->nb_queue_pools =
3999                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
4000                 vmdq_tx_conf->nb_queue_pools =
4001                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
4002
4003                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
4004                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
4005                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
4006                         vmdq_rx_conf->pool_map[i].pools =
4007                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
4008                 }
4009                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
4010                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
4011                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
4012                 }
4013
4014                 /* set DCB mode of RX and TX of multiple queues */
4015                 eth_conf->rxmode.mq_mode =
4016                                 (enum rte_eth_rx_mq_mode)
4017                                         (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
4018                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
4019         } else {
4020                 struct rte_eth_dcb_rx_conf *rx_conf =
4021                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
4022                 struct rte_eth_dcb_tx_conf *tx_conf =
4023                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
4024
4025                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
4026
4027                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
4028                 if (rc != 0)
4029                         return rc;
4030
4031                 rx_conf->nb_tcs = num_tcs;
4032                 tx_conf->nb_tcs = num_tcs;
4033
4034                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
4035                         rx_conf->dcb_tc[i] = i % num_tcs;
4036                         tx_conf->dcb_tc[i] = i % num_tcs;
4037                 }
4038
4039                 eth_conf->rxmode.mq_mode =
4040                                 (enum rte_eth_rx_mq_mode)
4041                                         (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
4042                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
4043                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
4044         }
4045
4046         if (pfc_en)
4047                 eth_conf->dcb_capability_en =
4048                                 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
4049         else
4050                 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
4051
4052         return 0;
4053 }
4054
4055 int
4056 init_port_dcb_config(portid_t pid,
4057                      enum dcb_mode_enable dcb_mode,
4058                      enum rte_eth_nb_tcs num_tcs,
4059                      uint8_t pfc_en)
4060 {
4061         struct rte_eth_conf port_conf;
4062         struct rte_port *rte_port;
4063         int retval;
4064         uint16_t i;
4065
4066         if (num_procs > 1) {
4067                 printf("The multi-process feature doesn't support dcb.\n");
4068                 return -ENOTSUP;
4069         }
4070         rte_port = &ports[pid];
4071
4072         /* retain the original device configuration. */
4073         memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
4074
4075         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
4076         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
4077         if (retval < 0)
4078                 return retval;
4079         port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4080         /* remove RSS HASH offload for DCB in vt mode */
4081         if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
4082                 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4083                 for (i = 0; i < nb_rxq; i++)
4084                         rte_port->rxq[i].conf.offloads &=
4085                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
4086         }
4087
4088         /* re-configure the device . */
4089         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
4090         if (retval < 0)
4091                 return retval;
4092
4093         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
4094         if (retval != 0)
4095                 return retval;
4096
4097         /* If dev_info.vmdq_pool_base is greater than 0,
4098          * the queue id of vmdq pools is started after pf queues.
4099          */
4100         if (dcb_mode == DCB_VT_ENABLED &&
4101             rte_port->dev_info.vmdq_pool_base > 0) {
4102                 fprintf(stderr,
4103                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
4104                         pid);
4105                 return -1;
4106         }
4107
4108         /* Assume the ports in testpmd have the same dcb capability
4109          * and has the same number of rxq and txq in dcb mode
4110          */
4111         if (dcb_mode == DCB_VT_ENABLED) {
4112                 if (rte_port->dev_info.max_vfs > 0) {
4113                         nb_rxq = rte_port->dev_info.nb_rx_queues;
4114                         nb_txq = rte_port->dev_info.nb_tx_queues;
4115                 } else {
4116                         nb_rxq = rte_port->dev_info.max_rx_queues;
4117                         nb_txq = rte_port->dev_info.max_tx_queues;
4118                 }
4119         } else {
4120                 /*if vt is disabled, use all pf queues */
4121                 if (rte_port->dev_info.vmdq_pool_base == 0) {
4122                         nb_rxq = rte_port->dev_info.max_rx_queues;
4123                         nb_txq = rte_port->dev_info.max_tx_queues;
4124                 } else {
4125                         nb_rxq = (queueid_t)num_tcs;
4126                         nb_txq = (queueid_t)num_tcs;
4127
4128                 }
4129         }
4130         rx_free_thresh = 64;
4131
4132         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4133
4134         rxtx_port_config(pid);
4135         /* VLAN filter */
4136         rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4137         for (i = 0; i < RTE_DIM(vlan_tags); i++)
4138                 rx_vft_set(pid, vlan_tags[i], 1);
4139
4140         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4141         if (retval != 0)
4142                 return retval;
4143
4144         rte_port->dcb_flag = 1;
4145
4146         /* Enter DCB configuration status */
4147         dcb_config = 1;
4148
4149         return 0;
4150 }
4151
4152 static void
4153 init_port(void)
4154 {
4155         int i;
4156
4157         /* Configuration of Ethernet ports. */
4158         ports = rte_zmalloc("testpmd: ports",
4159                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4160                             RTE_CACHE_LINE_SIZE);
4161         if (ports == NULL) {
4162                 rte_exit(EXIT_FAILURE,
4163                                 "rte_zmalloc(%d struct rte_port) failed\n",
4164                                 RTE_MAX_ETHPORTS);
4165         }
4166         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4167                 ports[i].xstats_info.allocated = false;
4168         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4169                 LIST_INIT(&ports[i].flow_tunnel_list);
4170         /* Initialize ports NUMA structures */
4171         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4172         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4173         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4174 }
4175
4176 static void
4177 force_quit(void)
4178 {
4179         pmd_test_exit();
4180         prompt_exit();
4181 }
4182
4183 static void
4184 print_stats(void)
4185 {
4186         uint8_t i;
4187         const char clr[] = { 27, '[', '2', 'J', '\0' };
4188         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4189
4190         /* Clear screen and move to top left */
4191         printf("%s%s", clr, top_left);
4192
4193         printf("\nPort statistics ====================================");
4194         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4195                 nic_stats_display(fwd_ports_ids[i]);
4196
4197         fflush(stdout);
4198 }
4199
4200 static void
4201 signal_handler(int signum)
4202 {
4203         if (signum == SIGINT || signum == SIGTERM) {
4204                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4205                         signum);
4206 #ifdef RTE_LIB_PDUMP
4207                 /* uninitialize packet capture framework */
4208                 rte_pdump_uninit();
4209 #endif
4210 #ifdef RTE_LIB_LATENCYSTATS
4211                 if (latencystats_enabled != 0)
4212                         rte_latencystats_uninit();
4213 #endif
4214                 force_quit();
4215                 /* Set flag to indicate the force termination. */
4216                 f_quit = 1;
4217                 /* exit with the expected status */
4218 #ifndef RTE_EXEC_ENV_WINDOWS
4219                 signal(signum, SIG_DFL);
4220                 kill(getpid(), signum);
4221 #endif
4222         }
4223 }
4224
4225 int
4226 main(int argc, char** argv)
4227 {
4228         int diag;
4229         portid_t port_id;
4230         uint16_t count;
4231         int ret;
4232
4233         signal(SIGINT, signal_handler);
4234         signal(SIGTERM, signal_handler);
4235
4236         testpmd_logtype = rte_log_register("testpmd");
4237         if (testpmd_logtype < 0)
4238                 rte_exit(EXIT_FAILURE, "Cannot register log type");
4239         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4240
4241         diag = rte_eal_init(argc, argv);
4242         if (diag < 0)
4243                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4244                          rte_strerror(rte_errno));
4245
4246         ret = register_eth_event_callback();
4247         if (ret != 0)
4248                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4249
4250 #ifdef RTE_LIB_PDUMP
4251         /* initialize packet capture framework */
4252         rte_pdump_init();
4253 #endif
4254
4255         count = 0;
4256         RTE_ETH_FOREACH_DEV(port_id) {
4257                 ports_ids[count] = port_id;
4258                 count++;
4259         }
4260         nb_ports = (portid_t) count;
4261         if (nb_ports == 0)
4262                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4263
4264         /* allocate port structures, and init them */
4265         init_port();
4266
4267         set_def_fwd_config();
4268         if (nb_lcores == 0)
4269                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4270                          "Check the core mask argument\n");
4271
4272         /* Bitrate/latency stats disabled by default */
4273 #ifdef RTE_LIB_BITRATESTATS
4274         bitrate_enabled = 0;
4275 #endif
4276 #ifdef RTE_LIB_LATENCYSTATS
4277         latencystats_enabled = 0;
4278 #endif
4279
4280         /* on FreeBSD, mlockall() is disabled by default */
4281 #ifdef RTE_EXEC_ENV_FREEBSD
4282         do_mlockall = 0;
4283 #else
4284         do_mlockall = 1;
4285 #endif
4286
4287         argc -= diag;
4288         argv += diag;
4289         if (argc > 1)
4290                 launch_args_parse(argc, argv);
4291
4292 #ifndef RTE_EXEC_ENV_WINDOWS
4293         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4294                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4295                         strerror(errno));
4296         }
4297 #endif
4298
4299         if (tx_first && interactive)
4300                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4301                                 "interactive mode.\n");
4302
4303         if (tx_first && lsc_interrupt) {
4304                 fprintf(stderr,
4305                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4306                 lsc_interrupt = 0;
4307         }
4308
4309         if (!nb_rxq && !nb_txq)
4310                 fprintf(stderr,
4311                         "Warning: Either rx or tx queues should be non-zero\n");
4312
4313         if (nb_rxq > 1 && nb_rxq > nb_txq)
4314                 fprintf(stderr,
4315                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4316                         nb_rxq, nb_txq);
4317
4318         init_config();
4319
4320         if (hot_plug) {
4321                 ret = rte_dev_hotplug_handle_enable();
4322                 if (ret) {
4323                         RTE_LOG(ERR, EAL,
4324                                 "fail to enable hotplug handling.");
4325                         return -1;
4326                 }
4327
4328                 ret = rte_dev_event_monitor_start();
4329                 if (ret) {
4330                         RTE_LOG(ERR, EAL,
4331                                 "fail to start device event monitoring.");
4332                         return -1;
4333                 }
4334
4335                 ret = rte_dev_event_callback_register(NULL,
4336                         dev_event_callback, NULL);
4337                 if (ret) {
4338                         RTE_LOG(ERR, EAL,
4339                                 "fail  to register device event callback\n");
4340                         return -1;
4341                 }
4342         }
4343
4344         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4345                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4346
4347         /* set all ports to promiscuous mode by default */
4348         RTE_ETH_FOREACH_DEV(port_id) {
4349                 ret = rte_eth_promiscuous_enable(port_id);
4350                 if (ret != 0)
4351                         fprintf(stderr,
4352                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4353                                 port_id, rte_strerror(-ret));
4354         }
4355
4356 #ifdef RTE_LIB_METRICS
4357         /* Init metrics library */
4358         rte_metrics_init(rte_socket_id());
4359 #endif
4360
4361 #ifdef RTE_LIB_LATENCYSTATS
4362         if (latencystats_enabled != 0) {
4363                 int ret = rte_latencystats_init(1, NULL);
4364                 if (ret)
4365                         fprintf(stderr,
4366                                 "Warning: latencystats init() returned error %d\n",
4367                                 ret);
4368                 fprintf(stderr, "Latencystats running on lcore %d\n",
4369                         latencystats_lcore_id);
4370         }
4371 #endif
4372
4373         /* Setup bitrate stats */
4374 #ifdef RTE_LIB_BITRATESTATS
4375         if (bitrate_enabled != 0) {
4376                 bitrate_data = rte_stats_bitrate_create();
4377                 if (bitrate_data == NULL)
4378                         rte_exit(EXIT_FAILURE,
4379                                 "Could not allocate bitrate data.\n");
4380                 rte_stats_bitrate_reg(bitrate_data);
4381         }
4382 #endif
4383 #ifdef RTE_LIB_CMDLINE
4384         if (init_cmdline() != 0)
4385                 rte_exit(EXIT_FAILURE,
4386                         "Could not initialise cmdline context.\n");
4387
4388         if (strlen(cmdline_filename) != 0)
4389                 cmdline_read_from_file(cmdline_filename);
4390
4391         if (interactive == 1) {
4392                 if (auto_start) {
4393                         printf("Start automatic packet forwarding\n");
4394                         start_packet_forwarding(0);
4395                 }
4396                 prompt();
4397                 pmd_test_exit();
4398         } else
4399 #endif
4400         {
4401                 char c;
4402                 int rc;
4403
4404                 f_quit = 0;
4405
4406                 printf("No commandline core given, start packet forwarding\n");
4407                 start_packet_forwarding(tx_first);
4408                 if (stats_period != 0) {
4409                         uint64_t prev_time = 0, cur_time, diff_time = 0;
4410                         uint64_t timer_period;
4411
4412                         /* Convert to number of cycles */
4413                         timer_period = stats_period * rte_get_timer_hz();
4414
4415                         while (f_quit == 0) {
4416                                 cur_time = rte_get_timer_cycles();
4417                                 diff_time += cur_time - prev_time;
4418
4419                                 if (diff_time >= timer_period) {
4420                                         print_stats();
4421                                         /* Reset the timer */
4422                                         diff_time = 0;
4423                                 }
4424                                 /* Sleep to avoid unnecessary checks */
4425                                 prev_time = cur_time;
4426                                 rte_delay_us_sleep(US_PER_S);
4427                         }
4428                 }
4429
4430                 printf("Press enter to exit\n");
4431                 rc = read(0, &c, 1);
4432                 pmd_test_exit();
4433                 if (rc < 0)
4434                         return 1;
4435         }
4436
4437         ret = rte_eal_cleanup();
4438         if (ret != 0)
4439                 rte_exit(EXIT_FAILURE,
4440                          "EAL cleanup failed: %s\n", strerror(-ret));
4441
4442         return EXIT_SUCCESS;
4443 }