app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_branch_prediction.h>
  40 #include <rte_mempool.h>
  41 #include <rte_malloc.h>
  42 #include <rte_mbuf.h>
  43 #include <rte_mbuf_pool_ops.h>
  44 #include <rte_interrupts.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_dev.h>
  49 #include <rte_string_fns.h>
  50 #ifdef RTE_NET_IXGBE
  51 #include <rte_pmd_ixgbe.h>
  52 #endif
  53 #ifdef RTE_LIB_PDUMP
  54 #include <rte_pdump.h>
  55 #endif
  56 #include <rte_flow.h>
  57 #ifdef RTE_LIB_METRICS
  58 #include <rte_metrics.h>
  59 #endif
  60 #ifdef RTE_LIB_BITRATESTATS
  61 #include <rte_bitrate.h>
  62 #endif
  63 #ifdef RTE_LIB_LATENCYSTATS
  64 #include <rte_latencystats.h>
  65 #endif
  66 #ifdef RTE_EXEC_ENV_WINDOWS
  67 #include <process.h>
  68 #endif
  69
  70 #include "testpmd.h"
  71
  72 #ifndef MAP_HUGETLB
  73 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  74 #define HUGE_FLAG (0x40000)
  75 #else
  76 #define HUGE_FLAG MAP_HUGETLB
  77 #endif
  78
  79 #ifndef MAP_HUGE_SHIFT
  80 /* older kernels (or FreeBSD) will not have this define */
  81 #define HUGE_SHIFT (26)
  82 #else
  83 #define HUGE_SHIFT MAP_HUGE_SHIFT
  84 #endif
  85
  86 #define EXTMEM_HEAP_NAME "extmem"
  87 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  88
  89 uint16_t verbose_level = 0; /**< Silent by default. */
  90 int testpmd_logtype; /**< Log type for testpmd logs */
  91
  92 /* use main core for command line ? */
  93 uint8_t interactive = 0;
  94 uint8_t auto_start = 0;
  95 uint8_t tx_first;
  96 char cmdline_filename[PATH_MAX] = {0};
  97
  98 /*
  99  * NUMA support configuration.
 100  * When set, the NUMA support attempts to dispatch the allocation of the
 101  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 102  * probed ports among the CPU sockets 0 and 1.
 103  * Otherwise, all memory is allocated from CPU socket 0.
 104  */
 105 uint8_t numa_support = 1; /**< numa enabled by default */
 106
 107 /*
 108  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 109  * not configured.
 110  */
 111 uint8_t socket_num = UMA_NO_CONFIG;
 112
 113 /*
 114  * Select mempool allocation type:
 115  * - native: use regular DPDK memory
 116  * - anon: use regular DPDK memory to create mempool, but populate using
 117  *         anonymous memory (may not be IOVA-contiguous)
 118  * - xmem: use externally allocated hugepage memory
 119  */
 120 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 121
 122 /*
 123  * Store specified sockets on which memory pool to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t port_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which RX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Store specified sockets on which TX ring to be used by ports
 136  * is allocated.
 137  */
 138 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 139
 140 /*
 141  * Record the Ethernet address of peer target ports to which packets are
 142  * forwarded.
 143  * Must be instantiated with the ethernet addresses of peer traffic generator
 144  * ports.
 145  */
 146 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 147 portid_t nb_peer_eth_addrs = 0;
 148
 149 /*
 150  * Probed Target Environment.
 151  */
 152 struct rte_port *ports;        /**< For all probed ethernet ports. */
 153 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 154 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 155 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 156
 157 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 158
 159 /*
 160  * Test Forwarding Configuration.
 161  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 162  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 163  */
 164 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 165 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 166 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 167 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 168
 169 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 170 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 171
 172 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 173 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 174
 175 /*
 176  * Forwarding engines.
 177  */
 178 struct fwd_engine * fwd_engines[] = {
 179         &io_fwd_engine,
 180         &mac_fwd_engine,
 181         &mac_swap_engine,
 182         &flow_gen_engine,
 183         &rx_only_engine,
 184         &tx_only_engine,
 185         &csum_fwd_engine,
 186         &icmp_echo_engine,
 187         &noisy_vnf_engine,
 188         &five_tuple_swap_fwd_engine,
 189 #ifdef RTE_LIBRTE_IEEE1588
 190         &ieee1588_fwd_engine,
 191 #endif
 192         &shared_rxq_engine,
 193         NULL,
 194 };
 195
 196 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 197 uint16_t mempool_flags;
 198
 199 struct fwd_config cur_fwd_config;
 200 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 201 uint32_t retry_enabled;
 202 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 203 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 204
 205 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 206 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 207         DEFAULT_MBUF_DATA_SIZE
 208 }; /**< Mbuf data space size. */
 209 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 210                                       * specified on command-line. */
 211 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 212
 213 /** Extended statistics to show. */
 214 struct rte_eth_xstat_name *xstats_display;
 215
 216 unsigned int xstats_display_num; /**< Size of extended statistics to show */
 217
 218 /*
 219  * In container, it cannot terminate the process which running with 'stats-period'
 220  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 221  */
 222 uint8_t f_quit;
 223
 224 /*
 225  * Max Rx frame size, set by '--max-pkt-len' parameter.
 226  */
 227 uint32_t max_rx_pkt_len;
 228
 229 /*
 230  * Configuration of packet segments used to scatter received packets
 231  * if some of split features is configured.
 232  */
 233 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 234 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 235 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 236 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 237
 238 /*
 239  * Configuration of packet segments used by the "txonly" processing engine.
 240  */
 241 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 242 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 243         TXONLY_DEF_PACKET_LEN,
 244 };
 245 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 246
 247 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 248 /**< Split policy for packets to TX. */
 249
 250 uint8_t txonly_multi_flow;
 251 /**< Whether multiple flows are generated in TXONLY mode. */
 252
 253 uint32_t tx_pkt_times_inter;
 254 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 255
 256 uint32_t tx_pkt_times_intra;
 257 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 258
 259 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 260 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 261 int nb_flows_flowgen = 1024; /**< Number of flows in flowgen mode. */
 262 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 263
 264 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 265 uint8_t dcb_config = 0;
 266
 267 /*
 268  * Configurable number of RX/TX queues.
 269  */
 270 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 271 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 272 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 273
 274 /*
 275  * Configurable number of RX/TX ring descriptors.
 276  * Defaults are supplied by drivers via ethdev.
 277  */
 278 #define RTE_TEST_RX_DESC_DEFAULT 0
 279 #define RTE_TEST_TX_DESC_DEFAULT 0
 280 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 281 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 282
 283 #define RTE_PMD_PARAM_UNSET -1
 284 /*
 285  * Configurable values of RX and TX ring threshold registers.
 286  */
 287
 288 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 289 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 290 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 291
 292 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 293 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 294 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 295
 296 /*
 297  * Configurable value of RX free threshold.
 298  */
 299 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 300
 301 /*
 302  * Configurable value of RX drop enable.
 303  */
 304 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 305
 306 /*
 307  * Configurable value of TX free threshold.
 308  */
 309 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 310
 311 /*
 312  * Configurable value of TX RS bit threshold.
 313  */
 314 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 315
 316 /*
 317  * Configurable value of buffered packets before sending.
 318  */
 319 uint16_t noisy_tx_sw_bufsz;
 320
 321 /*
 322  * Configurable value of packet buffer timeout.
 323  */
 324 uint16_t noisy_tx_sw_buf_flush_time;
 325
 326 /*
 327  * Configurable value for size of VNF internal memory area
 328  * used for simulating noisy neighbour behaviour
 329  */
 330 uint64_t noisy_lkup_mem_sz;
 331
 332 /*
 333  * Configurable value of number of random writes done in
 334  * VNF simulation memory area.
 335  */
 336 uint64_t noisy_lkup_num_writes;
 337
 338 /*
 339  * Configurable value of number of random reads done in
 340  * VNF simulation memory area.
 341  */
 342 uint64_t noisy_lkup_num_reads;
 343
 344 /*
 345  * Configurable value of number of random reads/writes done in
 346  * VNF simulation memory area.
 347  */
 348 uint64_t noisy_lkup_num_reads_writes;
 349
 350 /*
 351  * Receive Side Scaling (RSS) configuration.
 352  */
 353 uint64_t rss_hf = RTE_ETH_RSS_IP; /* RSS IP by default. */
 354
 355 /*
 356  * Port topology configuration
 357  */
 358 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 359
 360 /*
 361  * Avoids to flush all the RX streams before starts forwarding.
 362  */
 363 uint8_t no_flush_rx = 0; /* flush by default */
 364
 365 /*
 366  * Flow API isolated mode.
 367  */
 368 uint8_t flow_isolate_all;
 369
 370 /*
 371  * Avoids to check link status when starting/stopping a port.
 372  */
 373 uint8_t no_link_check = 0; /* check by default */
 374
 375 /*
 376  * Don't automatically start all ports in interactive mode.
 377  */
 378 uint8_t no_device_start = 0;
 379
 380 /*
 381  * Enable link status change notification
 382  */
 383 uint8_t lsc_interrupt = 1; /* enabled by default */
 384
 385 /*
 386  * Enable device removal notification.
 387  */
 388 uint8_t rmv_interrupt = 1; /* enabled by default */
 389
 390 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 391
 392 /* After attach, port setup is called on event or by iterator */
 393 bool setup_on_probe_event = true;
 394
 395 /* Clear ptypes on port initialization. */
 396 uint8_t clear_ptypes = true;
 397
 398 /* Hairpin ports configuration mode. */
 399 uint16_t hairpin_mode;
 400
 401 /* Pretty printing of ethdev events */
 402 static const char * const eth_event_desc[] = {
 403         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 404         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 405         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 406         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 407         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 408         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 409         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 410         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 411         [RTE_ETH_EVENT_NEW] = "device probed",
 412         [RTE_ETH_EVENT_DESTROY] = "device released",
 413         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 414         [RTE_ETH_EVENT_MAX] = NULL,
 415 };
 416
 417 /*
 418  * Display or mask ether events
 419  * Default to all events except VF_MBOX
 420  */
 421 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 422                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 423                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 424                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 425                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 426                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 427                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 428                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 429 /*
 430  * Decide if all memory are locked for performance.
 431  */
 432 int do_mlockall = 0;
 433
 434 /*
 435  * NIC bypass mode configuration options.
 436  */
 437
 438 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 439 /* The NIC bypass watchdog timeout. */
 440 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 441 #endif
 442
 443
 444 #ifdef RTE_LIB_LATENCYSTATS
 445
 446 /*
 447  * Set when latency stats is enabled in the commandline
 448  */
 449 uint8_t latencystats_enabled;
 450
 451 /*
 452  * Lcore ID to serive latency statistics.
 453  */
 454 lcoreid_t latencystats_lcore_id = -1;
 455
 456 #endif
 457
 458 /*
 459  * Ethernet device configuration.
 460  */
 461 struct rte_eth_rxmode rx_mode;
 462
 463 struct rte_eth_txmode tx_mode = {
 464         .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
 465 };
 466
 467 struct rte_eth_fdir_conf fdir_conf = {
 468         .mode = RTE_FDIR_MODE_NONE,
 469         .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
 470         .status = RTE_FDIR_REPORT_STATUS,
 471         .mask = {
 472                 .vlan_tci_mask = 0xFFEF,
 473                 .ipv4_mask     = {
 474                         .src_ip = 0xFFFFFFFF,
 475                         .dst_ip = 0xFFFFFFFF,
 476                 },
 477                 .ipv6_mask     = {
 478                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 479                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 480                 },
 481                 .src_port_mask = 0xFFFF,
 482                 .dst_port_mask = 0xFFFF,
 483                 .mac_addr_byte_mask = 0xFF,
 484                 .tunnel_type_mask = 1,
 485                 .tunnel_id_mask = 0xFFFFFFFF,
 486         },
 487         .drop_queue = 127,
 488 };
 489
 490 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 491
 492 /*
 493  * Display zero values by default for xstats
 494  */
 495 uint8_t xstats_hide_zero;
 496
 497 /*
 498  * Measure of CPU cycles disabled by default
 499  */
 500 uint8_t record_core_cycles;
 501
 502 /*
 503  * Display of RX and TX bursts disabled by default
 504  */
 505 uint8_t record_burst_stats;
 506
 507 /*
 508  * Number of ports per shared Rx queue group, 0 disable.
 509  */
 510 uint32_t rxq_share;
 511
 512 unsigned int num_sockets = 0;
 513 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 514
 515 #ifdef RTE_LIB_BITRATESTATS
 516 /* Bitrate statistics */
 517 struct rte_stats_bitrates *bitrate_data;
 518 lcoreid_t bitrate_lcore_id;
 519 uint8_t bitrate_enabled;
 520 #endif
 521
 522 #ifdef RTE_LIB_GRO
 523 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 524 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 525 #endif
 526
 527 /*
 528  * hexadecimal bitmask of RX mq mode can be enabled.
 529  */
 530 enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
 531
 532 /*
 533  * Used to set forced link speed
 534  */
 535 uint32_t eth_link_speed;
 536
 537 /*
 538  * ID of the current process in multi-process, used to
 539  * configure the queues to be polled.
 540  */
 541 int proc_id;
 542
 543 /*
 544  * Number of processes in multi-process, used to
 545  * configure the queues to be polled.
 546  */
 547 unsigned int num_procs = 1;
 548
 549 static void
 550 eth_rx_metadata_negotiate_mp(uint16_t port_id)
 551 {
 552         uint64_t rx_meta_features = 0;
 553         int ret;
 554
 555         if (!is_proc_primary())
 556                 return;
 557
 558         rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
 559         rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
 560         rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
 561
 562         ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
 563         if (ret == 0) {
 564                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
 565                         TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
 566                                     port_id);
 567                 }
 568
 569                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
 570                         TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
 571                                     port_id);
 572                 }
 573
 574                 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
 575                         TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
 576                                     port_id);
 577                 }
 578         } else if (ret != -ENOTSUP) {
 579                 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
 580                          port_id, rte_strerror(-ret));
 581         }
 582 }
 583
 584 static int
 585 eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 586                       const struct rte_eth_conf *dev_conf)
 587 {
 588         if (is_proc_primary())
 589                 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
 590                                         dev_conf);
 591         return 0;
 592 }
 593
 594 static int
 595 eth_dev_start_mp(uint16_t port_id)
 596 {
 597         if (is_proc_primary())
 598                 return rte_eth_dev_start(port_id);
 599
 600         return 0;
 601 }
 602
 603 static int
 604 eth_dev_stop_mp(uint16_t port_id)
 605 {
 606         if (is_proc_primary())
 607                 return rte_eth_dev_stop(port_id);
 608
 609         return 0;
 610 }
 611
 612 static void
 613 mempool_free_mp(struct rte_mempool *mp)
 614 {
 615         if (is_proc_primary())
 616                 rte_mempool_free(mp);
 617 }
 618
 619 static int
 620 eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
 621 {
 622         if (is_proc_primary())
 623                 return rte_eth_dev_set_mtu(port_id, mtu);
 624
 625         return 0;
 626 }
 627
 628 /* Forward function declarations */
 629 static void setup_attached_port(portid_t pi);
 630 static void check_all_ports_link_status(uint32_t port_mask);
 631 static int eth_event_callback(portid_t port_id,
 632                               enum rte_eth_event_type type,
 633                               void *param, void *ret_param);
 634 static void dev_event_callback(const char *device_name,
 635                                 enum rte_dev_event_type type,
 636                                 void *param);
 637 static void fill_xstats_display_info(void);
 638
 639 /*
 640  * Check if all the ports are started.
 641  * If yes, return positive value. If not, return zero.
 642  */
 643 static int all_ports_started(void);
 644
 645 #ifdef RTE_LIB_GSO
 646 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 647 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 648 #endif
 649
 650 /* Holds the registered mbuf dynamic flags names. */
 651 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 652
 653
 654 /*
 655  * Helper function to check if socket is already discovered.
 656  * If yes, return positive value. If not, return zero.
 657  */
 658 int
 659 new_socket_id(unsigned int socket_id)
 660 {
 661         unsigned int i;
 662
 663         for (i = 0; i < num_sockets; i++) {
 664                 if (socket_ids[i] == socket_id)
 665                         return 0;
 666         }
 667         return 1;
 668 }
 669
 670 /*
 671  * Setup default configuration.
 672  */
 673 static void
 674 set_default_fwd_lcores_config(void)
 675 {
 676         unsigned int i;
 677         unsigned int nb_lc;
 678         unsigned int sock_num;
 679
 680         nb_lc = 0;
 681         for (i = 0; i < RTE_MAX_LCORE; i++) {
 682                 if (!rte_lcore_is_enabled(i))
 683                         continue;
 684                 sock_num = rte_lcore_to_socket_id(i);
 685                 if (new_socket_id(sock_num)) {
 686                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 687                                 rte_exit(EXIT_FAILURE,
 688                                          "Total sockets greater than %u\n",
 689                                          RTE_MAX_NUMA_NODES);
 690                         }
 691                         socket_ids[num_sockets++] = sock_num;
 692                 }
 693                 if (i == rte_get_main_lcore())
 694                         continue;
 695                 fwd_lcores_cpuids[nb_lc++] = i;
 696         }
 697         nb_lcores = (lcoreid_t) nb_lc;
 698         nb_cfg_lcores = nb_lcores;
 699         nb_fwd_lcores = 1;
 700 }
 701
 702 static void
 703 set_def_peer_eth_addrs(void)
 704 {
 705         portid_t i;
 706
 707         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 708                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 709                 peer_eth_addrs[i].addr_bytes[5] = i;
 710         }
 711 }
 712
 713 static void
 714 set_default_fwd_ports_config(void)
 715 {
 716         portid_t pt_id;
 717         int i = 0;
 718
 719         RTE_ETH_FOREACH_DEV(pt_id) {
 720                 fwd_ports_ids[i++] = pt_id;
 721
 722                 /* Update sockets info according to the attached device */
 723                 int socket_id = rte_eth_dev_socket_id(pt_id);
 724                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 725                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 726                                 rte_exit(EXIT_FAILURE,
 727                                          "Total sockets greater than %u\n",
 728                                          RTE_MAX_NUMA_NODES);
 729                         }
 730                         socket_ids[num_sockets++] = socket_id;
 731                 }
 732         }
 733
 734         nb_cfg_ports = nb_ports;
 735         nb_fwd_ports = nb_ports;
 736 }
 737
 738 void
 739 set_def_fwd_config(void)
 740 {
 741         set_default_fwd_lcores_config();
 742         set_def_peer_eth_addrs();
 743         set_default_fwd_ports_config();
 744 }
 745
 746 #ifndef RTE_EXEC_ENV_WINDOWS
 747 /* extremely pessimistic estimation of memory required to create a mempool */
 748 static int
 749 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 750 {
 751         unsigned int n_pages, mbuf_per_pg, leftover;
 752         uint64_t total_mem, mbuf_mem, obj_sz;
 753
 754         /* there is no good way to predict how much space the mempool will
 755          * occupy because it will allocate chunks on the fly, and some of those
 756          * will come from default DPDK memory while some will come from our
 757          * external memory, so just assume 128MB will be enough for everyone.
 758          */
 759         uint64_t hdr_mem = 128 << 20;
 760
 761         /* account for possible non-contiguousness */
 762         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 763         if (obj_sz > pgsz) {
 764                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 765                 return -1;
 766         }
 767
 768         mbuf_per_pg = pgsz / obj_sz;
 769         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 770         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 771
 772         mbuf_mem = n_pages * pgsz;
 773
 774         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 775
 776         if (total_mem > SIZE_MAX) {
 777                 TESTPMD_LOG(ERR, "Memory size too big\n");
 778                 return -1;
 779         }
 780         *out = (size_t)total_mem;
 781
 782         return 0;
 783 }
 784
 785 static int
 786 pagesz_flags(uint64_t page_sz)
 787 {
 788         /* as per mmap() manpage, all page sizes are log2 of page size
 789          * shifted by MAP_HUGE_SHIFT
 790          */
 791         int log2 = rte_log2_u64(page_sz);
 792
 793         return (log2 << HUGE_SHIFT);
 794 }
 795
 796 static void *
 797 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 798 {
 799         void *addr;
 800         int flags;
 801
 802         /* allocate anonymous hugepages */
 803         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 804         if (huge)
 805                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 806
 807         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 808         if (addr == MAP_FAILED)
 809                 return NULL;
 810
 811         return addr;
 812 }
 813
 814 struct extmem_param {
 815         void *addr;
 816         size_t len;
 817         size_t pgsz;
 818         rte_iova_t *iova_table;
 819         unsigned int iova_table_len;
 820 };
 821
 822 static int
 823 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 824                 bool huge)
 825 {
 826         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 827                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 828         unsigned int cur_page, n_pages, pgsz_idx;
 829         size_t mem_sz, cur_pgsz;
 830         rte_iova_t *iovas = NULL;
 831         void *addr;
 832         int ret;
 833
 834         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 835                 /* skip anything that is too big */
 836                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 837                         continue;
 838
 839                 cur_pgsz = pgsizes[pgsz_idx];
 840
 841                 /* if we were told not to allocate hugepages, override */
 842                 if (!huge)
 843                         cur_pgsz = sysconf(_SC_PAGESIZE);
 844
 845                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 846                 if (ret < 0) {
 847                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 848                         return -1;
 849                 }
 850
 851                 /* allocate our memory */
 852                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 853
 854                 /* if we couldn't allocate memory with a specified page size,
 855                  * that doesn't mean we can't do it with other page sizes, so
 856                  * try another one.
 857                  */
 858                 if (addr == NULL)
 859                         continue;
 860
 861                 /* store IOVA addresses for every page in this memory area */
 862                 n_pages = mem_sz / cur_pgsz;
 863
 864                 iovas = malloc(sizeof(*iovas) * n_pages);
 865
 866                 if (iovas == NULL) {
 867                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 868                         goto fail;
 869                 }
 870                 /* lock memory if it's not huge pages */
 871                 if (!huge)
 872                         mlock(addr, mem_sz);
 873
 874                 /* populate IOVA addresses */
 875                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 876                         rte_iova_t iova;
 877                         size_t offset;
 878                         void *cur;
 879
 880                         offset = cur_pgsz * cur_page;
 881                         cur = RTE_PTR_ADD(addr, offset);
 882
 883                         /* touch the page before getting its IOVA */
 884                         *(volatile char *)cur = 0;
 885
 886                         iova = rte_mem_virt2iova(cur);
 887
 888                         iovas[cur_page] = iova;
 889                 }
 890
 891                 break;
 892         }
 893         /* if we couldn't allocate anything */
 894         if (iovas == NULL)
 895                 return -1;
 896
 897         param->addr = addr;
 898         param->len = mem_sz;
 899         param->pgsz = cur_pgsz;
 900         param->iova_table = iovas;
 901         param->iova_table_len = n_pages;
 902
 903         return 0;
 904 fail:
 905         if (iovas)
 906                 free(iovas);
 907         if (addr)
 908                 munmap(addr, mem_sz);
 909
 910         return -1;
 911 }
 912
 913 static int
 914 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 915 {
 916         struct extmem_param param;
 917         int socket_id, ret;
 918
 919         memset(&param, 0, sizeof(param));
 920
 921         /* check if our heap exists */
 922         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 923         if (socket_id < 0) {
 924                 /* create our heap */
 925                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 926                 if (ret < 0) {
 927                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 928                         return -1;
 929                 }
 930         }
 931
 932         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 933         if (ret < 0) {
 934                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 935                 return -1;
 936         }
 937
 938         /* we now have a valid memory area, so add it to heap */
 939         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 940                         param.addr, param.len, param.iova_table,
 941                         param.iova_table_len, param.pgsz);
 942
 943         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 944
 945         /* not needed any more */
 946         free(param.iova_table);
 947
 948         if (ret < 0) {
 949                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 950                 munmap(param.addr, param.len);
 951                 return -1;
 952         }
 953
 954         /* success */
 955
 956         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 957                         param.len >> 20);
 958
 959         return 0;
 960 }
 961 static void
 962 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 963              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 964 {
 965         uint16_t pid = 0;
 966         int ret;
 967
 968         RTE_ETH_FOREACH_DEV(pid) {
 969                 struct rte_eth_dev_info dev_info;
 970
 971                 ret = eth_dev_info_get_print_err(pid, &dev_info);
 972                 if (ret != 0) {
 973                         TESTPMD_LOG(DEBUG,
 974                                     "unable to get device info for port %d on addr 0x%p,"
 975                                     "mempool unmapping will not be performed\n",
 976                                     pid, memhdr->addr);
 977                         continue;
 978                 }
 979
 980                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
 981                 if (ret) {
 982                         TESTPMD_LOG(DEBUG,
 983                                     "unable to DMA unmap addr 0x%p "
 984                                     "for device %s\n",
 985                                     memhdr->addr, dev_info.device->name);
 986                 }
 987         }
 988         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 989         if (ret) {
 990                 TESTPMD_LOG(DEBUG,
 991                             "unable to un-register addr 0x%p\n", memhdr->addr);
 992         }
 993 }
 994
 995 static void
 996 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 997            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 998 {
 999         uint16_t pid = 0;
1000         size_t page_size = sysconf(_SC_PAGESIZE);
1001         int ret;
1002
1003         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1004                                   page_size);
1005         if (ret) {
1006                 TESTPMD_LOG(DEBUG,
1007                             "unable to register addr 0x%p\n", memhdr->addr);
1008                 return;
1009         }
1010         RTE_ETH_FOREACH_DEV(pid) {
1011                 struct rte_eth_dev_info dev_info;
1012
1013                 ret = eth_dev_info_get_print_err(pid, &dev_info);
1014                 if (ret != 0) {
1015                         TESTPMD_LOG(DEBUG,
1016                                     "unable to get device info for port %d on addr 0x%p,"
1017                                     "mempool mapping will not be performed\n",
1018                                     pid, memhdr->addr);
1019                         continue;
1020                 }
1021                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1022                 if (ret) {
1023                         TESTPMD_LOG(DEBUG,
1024                                     "unable to DMA map addr 0x%p "
1025                                     "for device %s\n",
1026                                     memhdr->addr, dev_info.device->name);
1027                 }
1028         }
1029 }
1030 #endif
1031
1032 static unsigned int
1033 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1034             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1035 {
1036         struct rte_pktmbuf_extmem *xmem;
1037         unsigned int ext_num, zone_num, elt_num;
1038         uint16_t elt_size;
1039
1040         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1041         elt_num = EXTBUF_ZONE_SIZE / elt_size;
1042         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1043
1044         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1045         if (xmem == NULL) {
1046                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1047                                  "external buffer descriptors\n");
1048                 *ext_mem = NULL;
1049                 return 0;
1050         }
1051         for (ext_num = 0; ext_num < zone_num; ext_num++) {
1052                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1053                 const struct rte_memzone *mz;
1054                 char mz_name[RTE_MEMZONE_NAMESIZE];
1055                 int ret;
1056
1057                 ret = snprintf(mz_name, sizeof(mz_name),
1058                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1059                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1060                         errno = ENAMETOOLONG;
1061                         ext_num = 0;
1062                         break;
1063                 }
1064                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
1065                                                  socket_id,
1066                                                  RTE_MEMZONE_IOVA_CONTIG |
1067                                                  RTE_MEMZONE_1GB |
1068                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
1069                                                  EXTBUF_ZONE_SIZE);
1070                 if (mz == NULL) {
1071                         /*
1072                          * The caller exits on external buffer creation
1073                          * error, so there is no need to free memzones.
1074                          */
1075                         errno = ENOMEM;
1076                         ext_num = 0;
1077                         break;
1078                 }
1079                 xseg->buf_ptr = mz->addr;
1080                 xseg->buf_iova = mz->iova;
1081                 xseg->buf_len = EXTBUF_ZONE_SIZE;
1082                 xseg->elt_size = elt_size;
1083         }
1084         if (ext_num == 0 && xmem != NULL) {
1085                 free(xmem);
1086                 xmem = NULL;
1087         }
1088         *ext_mem = xmem;
1089         return ext_num;
1090 }
1091
1092 /*
1093  * Configuration initialisation done once at init time.
1094  */
1095 static struct rte_mempool *
1096 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1097                  unsigned int socket_id, uint16_t size_idx)
1098 {
1099         char pool_name[RTE_MEMPOOL_NAMESIZE];
1100         struct rte_mempool *rte_mp = NULL;
1101 #ifndef RTE_EXEC_ENV_WINDOWS
1102         uint32_t mb_size;
1103
1104         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1105 #endif
1106         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1107         if (!is_proc_primary()) {
1108                 rte_mp = rte_mempool_lookup(pool_name);
1109                 if (rte_mp == NULL)
1110                         rte_exit(EXIT_FAILURE,
1111                                 "Get mbuf pool for socket %u failed: %s\n",
1112                                 socket_id, rte_strerror(rte_errno));
1113                 return rte_mp;
1114         }
1115
1116         TESTPMD_LOG(INFO,
1117                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1118                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1119
1120         switch (mp_alloc_type) {
1121         case MP_ALLOC_NATIVE:
1122                 {
1123                         /* wrapper to rte_mempool_create() */
1124                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1125                                         rte_mbuf_best_mempool_ops());
1126                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1127                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1128                         break;
1129                 }
1130 #ifndef RTE_EXEC_ENV_WINDOWS
1131         case MP_ALLOC_ANON:
1132                 {
1133                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1134                                 mb_size, (unsigned int) mb_mempool_cache,
1135                                 sizeof(struct rte_pktmbuf_pool_private),
1136                                 socket_id, mempool_flags);
1137                         if (rte_mp == NULL)
1138                                 goto err;
1139
1140                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1141                                 rte_mempool_free(rte_mp);
1142                                 rte_mp = NULL;
1143                                 goto err;
1144                         }
1145                         rte_pktmbuf_pool_init(rte_mp, NULL);
1146                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1147                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1148                         break;
1149                 }
1150         case MP_ALLOC_XMEM:
1151         case MP_ALLOC_XMEM_HUGE:
1152                 {
1153                         int heap_socket;
1154                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1155
1156                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1157                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1158
1159                         heap_socket =
1160                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1161                         if (heap_socket < 0)
1162                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1163
1164                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1165                                         rte_mbuf_best_mempool_ops());
1166                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1167                                         mb_mempool_cache, 0, mbuf_seg_size,
1168                                         heap_socket);
1169                         break;
1170                 }
1171 #endif
1172         case MP_ALLOC_XBUF:
1173                 {
1174                         struct rte_pktmbuf_extmem *ext_mem;
1175                         unsigned int ext_num;
1176
1177                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1178                                                socket_id, pool_name, &ext_mem);
1179                         if (ext_num == 0)
1180                                 rte_exit(EXIT_FAILURE,
1181                                          "Can't create pinned data buffers\n");
1182
1183                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1184                                         rte_mbuf_best_mempool_ops());
1185                         rte_mp = rte_pktmbuf_pool_create_extbuf
1186                                         (pool_name, nb_mbuf, mb_mempool_cache,
1187                                          0, mbuf_seg_size, socket_id,
1188                                          ext_mem, ext_num);
1189                         free(ext_mem);
1190                         break;
1191                 }
1192         default:
1193                 {
1194                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1195                 }
1196         }
1197
1198 #ifndef RTE_EXEC_ENV_WINDOWS
1199 err:
1200 #endif
1201         if (rte_mp == NULL) {
1202                 rte_exit(EXIT_FAILURE,
1203                         "Creation of mbuf pool for socket %u failed: %s\n",
1204                         socket_id, rte_strerror(rte_errno));
1205         } else if (verbose_level > 0) {
1206                 rte_mempool_dump(stdout, rte_mp);
1207         }
1208         return rte_mp;
1209 }
1210
1211 /*
1212  * Check given socket id is valid or not with NUMA mode,
1213  * if valid, return 0, else return -1
1214  */
1215 static int
1216 check_socket_id(const unsigned int socket_id)
1217 {
1218         static int warning_once = 0;
1219
1220         if (new_socket_id(socket_id)) {
1221                 if (!warning_once && numa_support)
1222                         fprintf(stderr,
1223                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1224                 warning_once = 1;
1225                 return -1;
1226         }
1227         return 0;
1228 }
1229
1230 /*
1231  * Get the allowed maximum number of RX queues.
1232  * *pid return the port id which has minimal value of
1233  * max_rx_queues in all ports.
1234  */
1235 queueid_t
1236 get_allowed_max_nb_rxq(portid_t *pid)
1237 {
1238         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1239         bool max_rxq_valid = false;
1240         portid_t pi;
1241         struct rte_eth_dev_info dev_info;
1242
1243         RTE_ETH_FOREACH_DEV(pi) {
1244                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1245                         continue;
1246
1247                 max_rxq_valid = true;
1248                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1249                         allowed_max_rxq = dev_info.max_rx_queues;
1250                         *pid = pi;
1251                 }
1252         }
1253         return max_rxq_valid ? allowed_max_rxq : 0;
1254 }
1255
1256 /*
1257  * Check input rxq is valid or not.
1258  * If input rxq is not greater than any of maximum number
1259  * of RX queues of all ports, it is valid.
1260  * if valid, return 0, else return -1
1261  */
1262 int
1263 check_nb_rxq(queueid_t rxq)
1264 {
1265         queueid_t allowed_max_rxq;
1266         portid_t pid = 0;
1267
1268         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1269         if (rxq > allowed_max_rxq) {
1270                 fprintf(stderr,
1271                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1272                         rxq, allowed_max_rxq, pid);
1273                 return -1;
1274         }
1275         return 0;
1276 }
1277
1278 /*
1279  * Get the allowed maximum number of TX queues.
1280  * *pid return the port id which has minimal value of
1281  * max_tx_queues in all ports.
1282  */
1283 queueid_t
1284 get_allowed_max_nb_txq(portid_t *pid)
1285 {
1286         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1287         bool max_txq_valid = false;
1288         portid_t pi;
1289         struct rte_eth_dev_info dev_info;
1290
1291         RTE_ETH_FOREACH_DEV(pi) {
1292                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1293                         continue;
1294
1295                 max_txq_valid = true;
1296                 if (dev_info.max_tx_queues < allowed_max_txq) {
1297                         allowed_max_txq = dev_info.max_tx_queues;
1298                         *pid = pi;
1299                 }
1300         }
1301         return max_txq_valid ? allowed_max_txq : 0;
1302 }
1303
1304 /*
1305  * Check input txq is valid or not.
1306  * If input txq is not greater than any of maximum number
1307  * of TX queues of all ports, it is valid.
1308  * if valid, return 0, else return -1
1309  */
1310 int
1311 check_nb_txq(queueid_t txq)
1312 {
1313         queueid_t allowed_max_txq;
1314         portid_t pid = 0;
1315
1316         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1317         if (txq > allowed_max_txq) {
1318                 fprintf(stderr,
1319                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1320                         txq, allowed_max_txq, pid);
1321                 return -1;
1322         }
1323         return 0;
1324 }
1325
1326 /*
1327  * Get the allowed maximum number of RXDs of every rx queue.
1328  * *pid return the port id which has minimal value of
1329  * max_rxd in all queues of all ports.
1330  */
1331 static uint16_t
1332 get_allowed_max_nb_rxd(portid_t *pid)
1333 {
1334         uint16_t allowed_max_rxd = UINT16_MAX;
1335         portid_t pi;
1336         struct rte_eth_dev_info dev_info;
1337
1338         RTE_ETH_FOREACH_DEV(pi) {
1339                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1340                         continue;
1341
1342                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1343                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1344                         *pid = pi;
1345                 }
1346         }
1347         return allowed_max_rxd;
1348 }
1349
1350 /*
1351  * Get the allowed minimal number of RXDs of every rx queue.
1352  * *pid return the port id which has minimal value of
1353  * min_rxd in all queues of all ports.
1354  */
1355 static uint16_t
1356 get_allowed_min_nb_rxd(portid_t *pid)
1357 {
1358         uint16_t allowed_min_rxd = 0;
1359         portid_t pi;
1360         struct rte_eth_dev_info dev_info;
1361
1362         RTE_ETH_FOREACH_DEV(pi) {
1363                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1364                         continue;
1365
1366                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1367                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1368                         *pid = pi;
1369                 }
1370         }
1371
1372         return allowed_min_rxd;
1373 }
1374
1375 /*
1376  * Check input rxd is valid or not.
1377  * If input rxd is not greater than any of maximum number
1378  * of RXDs of every Rx queues and is not less than any of
1379  * minimal number of RXDs of every Rx queues, it is valid.
1380  * if valid, return 0, else return -1
1381  */
1382 int
1383 check_nb_rxd(queueid_t rxd)
1384 {
1385         uint16_t allowed_max_rxd;
1386         uint16_t allowed_min_rxd;
1387         portid_t pid = 0;
1388
1389         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1390         if (rxd > allowed_max_rxd) {
1391                 fprintf(stderr,
1392                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1393                         rxd, allowed_max_rxd, pid);
1394                 return -1;
1395         }
1396
1397         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1398         if (rxd < allowed_min_rxd) {
1399                 fprintf(stderr,
1400                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1401                         rxd, allowed_min_rxd, pid);
1402                 return -1;
1403         }
1404
1405         return 0;
1406 }
1407
1408 /*
1409  * Get the allowed maximum number of TXDs of every rx queues.
1410  * *pid return the port id which has minimal value of
1411  * max_txd in every tx queue.
1412  */
1413 static uint16_t
1414 get_allowed_max_nb_txd(portid_t *pid)
1415 {
1416         uint16_t allowed_max_txd = UINT16_MAX;
1417         portid_t pi;
1418         struct rte_eth_dev_info dev_info;
1419
1420         RTE_ETH_FOREACH_DEV(pi) {
1421                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1422                         continue;
1423
1424                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1425                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1426                         *pid = pi;
1427                 }
1428         }
1429         return allowed_max_txd;
1430 }
1431
1432 /*
1433  * Get the allowed maximum number of TXDs of every tx queues.
1434  * *pid return the port id which has minimal value of
1435  * min_txd in every tx queue.
1436  */
1437 static uint16_t
1438 get_allowed_min_nb_txd(portid_t *pid)
1439 {
1440         uint16_t allowed_min_txd = 0;
1441         portid_t pi;
1442         struct rte_eth_dev_info dev_info;
1443
1444         RTE_ETH_FOREACH_DEV(pi) {
1445                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1446                         continue;
1447
1448                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1449                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1450                         *pid = pi;
1451                 }
1452         }
1453
1454         return allowed_min_txd;
1455 }
1456
1457 /*
1458  * Check input txd is valid or not.
1459  * If input txd is not greater than any of maximum number
1460  * of TXDs of every Rx queues, it is valid.
1461  * if valid, return 0, else return -1
1462  */
1463 int
1464 check_nb_txd(queueid_t txd)
1465 {
1466         uint16_t allowed_max_txd;
1467         uint16_t allowed_min_txd;
1468         portid_t pid = 0;
1469
1470         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1471         if (txd > allowed_max_txd) {
1472                 fprintf(stderr,
1473                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1474                         txd, allowed_max_txd, pid);
1475                 return -1;
1476         }
1477
1478         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1479         if (txd < allowed_min_txd) {
1480                 fprintf(stderr,
1481                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1482                         txd, allowed_min_txd, pid);
1483                 return -1;
1484         }
1485         return 0;
1486 }
1487
1488
1489 /*
1490  * Get the allowed maximum number of hairpin queues.
1491  * *pid return the port id which has minimal value of
1492  * max_hairpin_queues in all ports.
1493  */
1494 queueid_t
1495 get_allowed_max_nb_hairpinq(portid_t *pid)
1496 {
1497         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1498         portid_t pi;
1499         struct rte_eth_hairpin_cap cap;
1500
1501         RTE_ETH_FOREACH_DEV(pi) {
1502                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1503                         *pid = pi;
1504                         return 0;
1505                 }
1506                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1507                         allowed_max_hairpinq = cap.max_nb_queues;
1508                         *pid = pi;
1509                 }
1510         }
1511         return allowed_max_hairpinq;
1512 }
1513
1514 /*
1515  * Check input hairpin is valid or not.
1516  * If input hairpin is not greater than any of maximum number
1517  * of hairpin queues of all ports, it is valid.
1518  * if valid, return 0, else return -1
1519  */
1520 int
1521 check_nb_hairpinq(queueid_t hairpinq)
1522 {
1523         queueid_t allowed_max_hairpinq;
1524         portid_t pid = 0;
1525
1526         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1527         if (hairpinq > allowed_max_hairpinq) {
1528                 fprintf(stderr,
1529                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1530                         hairpinq, allowed_max_hairpinq, pid);
1531                 return -1;
1532         }
1533         return 0;
1534 }
1535
1536 static int
1537 get_eth_overhead(struct rte_eth_dev_info *dev_info)
1538 {
1539         uint32_t eth_overhead;
1540
1541         if (dev_info->max_mtu != UINT16_MAX &&
1542             dev_info->max_rx_pktlen > dev_info->max_mtu)
1543                 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1544         else
1545                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1546
1547         return eth_overhead;
1548 }
1549
1550 static void
1551 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1552 {
1553         struct rte_port *port = &ports[pid];
1554         int ret;
1555         int i;
1556
1557         eth_rx_metadata_negotiate_mp(pid);
1558
1559         port->dev_conf.txmode = tx_mode;
1560         port->dev_conf.rxmode = rx_mode;
1561
1562         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1563         if (ret != 0)
1564                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1565
1566         if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1567                 port->dev_conf.txmode.offloads &=
1568                         ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1569
1570         /* Apply Rx offloads configuration */
1571         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1572                 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1573         /* Apply Tx offloads configuration */
1574         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1575                 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1576
1577         if (eth_link_speed)
1578                 port->dev_conf.link_speeds = eth_link_speed;
1579
1580         if (max_rx_pkt_len)
1581                 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1582                         get_eth_overhead(&port->dev_info);
1583
1584         /* set flag to initialize port/queue */
1585         port->need_reconfig = 1;
1586         port->need_reconfig_queues = 1;
1587         port->socket_id = socket_id;
1588         port->tx_metadata = 0;
1589
1590         /*
1591          * Check for maximum number of segments per MTU.
1592          * Accordingly update the mbuf data size.
1593          */
1594         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1595             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1596                 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1597                 uint16_t mtu;
1598
1599                 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1600                         uint16_t data_size = (mtu + eth_overhead) /
1601                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1602                         uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1603
1604                         if (buffer_size > mbuf_data_size[0]) {
1605                                 mbuf_data_size[0] = buffer_size;
1606                                 TESTPMD_LOG(WARNING,
1607                                         "Configured mbuf size of the first segment %hu\n",
1608                                         mbuf_data_size[0]);
1609                         }
1610                 }
1611         }
1612 }
1613
1614 static void
1615 init_config(void)
1616 {
1617         portid_t pid;
1618         struct rte_mempool *mbp;
1619         unsigned int nb_mbuf_per_pool;
1620         lcoreid_t  lc_id;
1621 #ifdef RTE_LIB_GRO
1622         struct rte_gro_param gro_param;
1623 #endif
1624 #ifdef RTE_LIB_GSO
1625         uint32_t gso_types;
1626 #endif
1627
1628         /* Configuration of logical cores. */
1629         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1630                                 sizeof(struct fwd_lcore *) * nb_lcores,
1631                                 RTE_CACHE_LINE_SIZE);
1632         if (fwd_lcores == NULL) {
1633                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1634                                                         "failed\n", nb_lcores);
1635         }
1636         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1637                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1638                                                sizeof(struct fwd_lcore),
1639                                                RTE_CACHE_LINE_SIZE);
1640                 if (fwd_lcores[lc_id] == NULL) {
1641                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1642                                                                 "failed\n");
1643                 }
1644                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1645         }
1646
1647         RTE_ETH_FOREACH_DEV(pid) {
1648                 uint32_t socket_id;
1649
1650                 if (numa_support) {
1651                         socket_id = port_numa[pid];
1652                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1653                                 socket_id = rte_eth_dev_socket_id(pid);
1654
1655                                 /*
1656                                  * if socket_id is invalid,
1657                                  * set to the first available socket.
1658                                  */
1659                                 if (check_socket_id(socket_id) < 0)
1660                                         socket_id = socket_ids[0];
1661                         }
1662                 } else {
1663                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1664                                     0 : socket_num;
1665                 }
1666                 /* Apply default TxRx configuration for all ports */
1667                 init_config_port_offloads(pid, socket_id);
1668         }
1669         /*
1670          * Create pools of mbuf.
1671          * If NUMA support is disabled, create a single pool of mbuf in
1672          * socket 0 memory by default.
1673          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1674          *
1675          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1676          * nb_txd can be configured at run time.
1677          */
1678         if (param_total_num_mbufs)
1679                 nb_mbuf_per_pool = param_total_num_mbufs;
1680         else {
1681                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1682                         (nb_lcores * mb_mempool_cache) +
1683                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1684                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1685         }
1686
1687         if (numa_support) {
1688                 uint8_t i, j;
1689
1690                 for (i = 0; i < num_sockets; i++)
1691                         for (j = 0; j < mbuf_data_size_n; j++)
1692                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1693                                         mbuf_pool_create(mbuf_data_size[j],
1694                                                           nb_mbuf_per_pool,
1695                                                           socket_ids[i], j);
1696         } else {
1697                 uint8_t i;
1698
1699                 for (i = 0; i < mbuf_data_size_n; i++)
1700                         mempools[i] = mbuf_pool_create
1701                                         (mbuf_data_size[i],
1702                                          nb_mbuf_per_pool,
1703                                          socket_num == UMA_NO_CONFIG ?
1704                                          0 : socket_num, i);
1705         }
1706
1707         init_port_config();
1708
1709 #ifdef RTE_LIB_GSO
1710         gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1711                 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1712 #endif
1713         /*
1714          * Records which Mbuf pool to use by each logical core, if needed.
1715          */
1716         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1717                 mbp = mbuf_pool_find(
1718                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1719
1720                 if (mbp == NULL)
1721                         mbp = mbuf_pool_find(0, 0);
1722                 fwd_lcores[lc_id]->mbp = mbp;
1723 #ifdef RTE_LIB_GSO
1724                 /* initialize GSO context */
1725                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1726                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1727                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1728                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1729                         RTE_ETHER_CRC_LEN;
1730                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1731 #endif
1732         }
1733
1734         fwd_config_setup();
1735
1736 #ifdef RTE_LIB_GRO
1737         /* create a gro context for each lcore */
1738         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1739         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1740         gro_param.max_item_per_flow = MAX_PKT_BURST;
1741         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1742                 gro_param.socket_id = rte_lcore_to_socket_id(
1743                                 fwd_lcores_cpuids[lc_id]);
1744                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1745                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1746                         rte_exit(EXIT_FAILURE,
1747                                         "rte_gro_ctx_create() failed\n");
1748                 }
1749         }
1750 #endif
1751 }
1752
1753
1754 void
1755 reconfig(portid_t new_port_id, unsigned socket_id)
1756 {
1757         /* Reconfiguration of Ethernet ports. */
1758         init_config_port_offloads(new_port_id, socket_id);
1759         init_port_config();
1760 }
1761
1762
1763 int
1764 init_fwd_streams(void)
1765 {
1766         portid_t pid;
1767         struct rte_port *port;
1768         streamid_t sm_id, nb_fwd_streams_new;
1769         queueid_t q;
1770
1771         /* set socket id according to numa or not */
1772         RTE_ETH_FOREACH_DEV(pid) {
1773                 port = &ports[pid];
1774                 if (nb_rxq > port->dev_info.max_rx_queues) {
1775                         fprintf(stderr,
1776                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1777                                 nb_rxq, port->dev_info.max_rx_queues);
1778                         return -1;
1779                 }
1780                 if (nb_txq > port->dev_info.max_tx_queues) {
1781                         fprintf(stderr,
1782                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1783                                 nb_txq, port->dev_info.max_tx_queues);
1784                         return -1;
1785                 }
1786                 if (numa_support) {
1787                         if (port_numa[pid] != NUMA_NO_CONFIG)
1788                                 port->socket_id = port_numa[pid];
1789                         else {
1790                                 port->socket_id = rte_eth_dev_socket_id(pid);
1791
1792                                 /*
1793                                  * if socket_id is invalid,
1794                                  * set to the first available socket.
1795                                  */
1796                                 if (check_socket_id(port->socket_id) < 0)
1797                                         port->socket_id = socket_ids[0];
1798                         }
1799                 }
1800                 else {
1801                         if (socket_num == UMA_NO_CONFIG)
1802                                 port->socket_id = 0;
1803                         else
1804                                 port->socket_id = socket_num;
1805                 }
1806         }
1807
1808         q = RTE_MAX(nb_rxq, nb_txq);
1809         if (q == 0) {
1810                 fprintf(stderr,
1811                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1812                 return -1;
1813         }
1814         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1815         if (nb_fwd_streams_new == nb_fwd_streams)
1816                 return 0;
1817         /* clear the old */
1818         if (fwd_streams != NULL) {
1819                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1820                         if (fwd_streams[sm_id] == NULL)
1821                                 continue;
1822                         rte_free(fwd_streams[sm_id]);
1823                         fwd_streams[sm_id] = NULL;
1824                 }
1825                 rte_free(fwd_streams);
1826                 fwd_streams = NULL;
1827         }
1828
1829         /* init new */
1830         nb_fwd_streams = nb_fwd_streams_new;
1831         if (nb_fwd_streams) {
1832                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1833                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1834                         RTE_CACHE_LINE_SIZE);
1835                 if (fwd_streams == NULL)
1836                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1837                                  " (struct fwd_stream *)) failed\n",
1838                                  nb_fwd_streams);
1839
1840                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1841                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1842                                 " struct fwd_stream", sizeof(struct fwd_stream),
1843                                 RTE_CACHE_LINE_SIZE);
1844                         if (fwd_streams[sm_id] == NULL)
1845                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1846                                          "(struct fwd_stream) failed\n");
1847                 }
1848         }
1849
1850         return 0;
1851 }
1852
1853 static void
1854 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1855 {
1856         uint64_t total_burst, sburst;
1857         uint64_t nb_burst;
1858         uint64_t burst_stats[4];
1859         uint16_t pktnb_stats[4];
1860         uint16_t nb_pkt;
1861         int burst_percent[4], sburstp;
1862         int i;
1863
1864         /*
1865          * First compute the total number of packet bursts and the
1866          * two highest numbers of bursts of the same number of packets.
1867          */
1868         memset(&burst_stats, 0x0, sizeof(burst_stats));
1869         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1870
1871         /* Show stats for 0 burst size always */
1872         total_burst = pbs->pkt_burst_spread[0];
1873         burst_stats[0] = pbs->pkt_burst_spread[0];
1874         pktnb_stats[0] = 0;
1875
1876         /* Find the next 2 burst sizes with highest occurrences. */
1877         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1878                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1879
1880                 if (nb_burst == 0)
1881                         continue;
1882
1883                 total_burst += nb_burst;
1884
1885                 if (nb_burst > burst_stats[1]) {
1886                         burst_stats[2] = burst_stats[1];
1887                         pktnb_stats[2] = pktnb_stats[1];
1888                         burst_stats[1] = nb_burst;
1889                         pktnb_stats[1] = nb_pkt;
1890                 } else if (nb_burst > burst_stats[2]) {
1891                         burst_stats[2] = nb_burst;
1892                         pktnb_stats[2] = nb_pkt;
1893                 }
1894         }
1895         if (total_burst == 0)
1896                 return;
1897
1898         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1899         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1900                 if (i == 3) {
1901                         printf("%d%% of other]\n", 100 - sburstp);
1902                         return;
1903                 }
1904
1905                 sburst += burst_stats[i];
1906                 if (sburst == total_burst) {
1907                         printf("%d%% of %d pkts]\n",
1908                                 100 - sburstp, (int) pktnb_stats[i]);
1909                         return;
1910                 }
1911
1912                 burst_percent[i] =
1913                         (double)burst_stats[i] / total_burst * 100;
1914                 printf("%d%% of %d pkts + ",
1915                         burst_percent[i], (int) pktnb_stats[i]);
1916                 sburstp += burst_percent[i];
1917         }
1918 }
1919
1920 static void
1921 fwd_stream_stats_display(streamid_t stream_id)
1922 {
1923         struct fwd_stream *fs;
1924         static const char *fwd_top_stats_border = "-------";
1925
1926         fs = fwd_streams[stream_id];
1927         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1928             (fs->fwd_dropped == 0))
1929                 return;
1930         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1931                "TX Port=%2d/Queue=%2d %s\n",
1932                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1933                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1934         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1935                " TX-dropped: %-14"PRIu64,
1936                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1937
1938         /* if checksum mode */
1939         if (cur_fwd_eng == &csum_fwd_engine) {
1940                 printf("  RX- bad IP checksum: %-14"PRIu64
1941                        "  Rx- bad L4 checksum: %-14"PRIu64
1942                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1943                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1944                         fs->rx_bad_outer_l4_csum);
1945                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1946                         fs->rx_bad_outer_ip_csum);
1947         } else {
1948                 printf("\n");
1949         }
1950
1951         if (record_burst_stats) {
1952                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1953                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1954         }
1955 }
1956
1957 void
1958 fwd_stats_display(void)
1959 {
1960         static const char *fwd_stats_border = "----------------------";
1961         static const char *acc_stats_border = "+++++++++++++++";
1962         struct {
1963                 struct fwd_stream *rx_stream;
1964                 struct fwd_stream *tx_stream;
1965                 uint64_t tx_dropped;
1966                 uint64_t rx_bad_ip_csum;
1967                 uint64_t rx_bad_l4_csum;
1968                 uint64_t rx_bad_outer_l4_csum;
1969                 uint64_t rx_bad_outer_ip_csum;
1970         } ports_stats[RTE_MAX_ETHPORTS];
1971         uint64_t total_rx_dropped = 0;
1972         uint64_t total_tx_dropped = 0;
1973         uint64_t total_rx_nombuf = 0;
1974         struct rte_eth_stats stats;
1975         uint64_t fwd_cycles = 0;
1976         uint64_t total_recv = 0;
1977         uint64_t total_xmit = 0;
1978         struct rte_port *port;
1979         streamid_t sm_id;
1980         portid_t pt_id;
1981         int i;
1982
1983         memset(ports_stats, 0, sizeof(ports_stats));
1984
1985         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1986                 struct fwd_stream *fs = fwd_streams[sm_id];
1987
1988                 if (cur_fwd_config.nb_fwd_streams >
1989                     cur_fwd_config.nb_fwd_ports) {
1990                         fwd_stream_stats_display(sm_id);
1991                 } else {
1992                         ports_stats[fs->tx_port].tx_stream = fs;
1993                         ports_stats[fs->rx_port].rx_stream = fs;
1994                 }
1995
1996                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1997
1998                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1999                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2000                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2001                                 fs->rx_bad_outer_l4_csum;
2002                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2003                                 fs->rx_bad_outer_ip_csum;
2004
2005                 if (record_core_cycles)
2006                         fwd_cycles += fs->core_cycles;
2007         }
2008         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2009                 pt_id = fwd_ports_ids[i];
2010                 port = &ports[pt_id];
2011
2012                 rte_eth_stats_get(pt_id, &stats);
2013                 stats.ipackets -= port->stats.ipackets;
2014                 stats.opackets -= port->stats.opackets;
2015                 stats.ibytes -= port->stats.ibytes;
2016                 stats.obytes -= port->stats.obytes;
2017                 stats.imissed -= port->stats.imissed;
2018                 stats.oerrors -= port->stats.oerrors;
2019                 stats.rx_nombuf -= port->stats.rx_nombuf;
2020
2021                 total_recv += stats.ipackets;
2022                 total_xmit += stats.opackets;
2023                 total_rx_dropped += stats.imissed;
2024                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2025                 total_tx_dropped += stats.oerrors;
2026                 total_rx_nombuf  += stats.rx_nombuf;
2027
2028                 printf("\n  %s Forward statistics for port %-2d %s\n",
2029                        fwd_stats_border, pt_id, fwd_stats_border);
2030
2031                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2032                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2033                        stats.ipackets + stats.imissed);
2034
2035                 if (cur_fwd_eng == &csum_fwd_engine) {
2036                         printf("  Bad-ipcsum: %-14"PRIu64
2037                                " Bad-l4csum: %-14"PRIu64
2038                                "Bad-outer-l4csum: %-14"PRIu64"\n",
2039                                ports_stats[pt_id].rx_bad_ip_csum,
2040                                ports_stats[pt_id].rx_bad_l4_csum,
2041                                ports_stats[pt_id].rx_bad_outer_l4_csum);
2042                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
2043                                ports_stats[pt_id].rx_bad_outer_ip_csum);
2044                 }
2045                 if (stats.ierrors + stats.rx_nombuf > 0) {
2046                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
2047                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2048                 }
2049
2050                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2051                        "TX-total: %-"PRIu64"\n",
2052                        stats.opackets, ports_stats[pt_id].tx_dropped,
2053                        stats.opackets + ports_stats[pt_id].tx_dropped);
2054
2055                 if (record_burst_stats) {
2056                         if (ports_stats[pt_id].rx_stream)
2057                                 pkt_burst_stats_display("RX",
2058                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
2059                         if (ports_stats[pt_id].tx_stream)
2060                                 pkt_burst_stats_display("TX",
2061                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2062                 }
2063
2064                 printf("  %s--------------------------------%s\n",
2065                        fwd_stats_border, fwd_stats_border);
2066         }
2067
2068         printf("\n  %s Accumulated forward statistics for all ports"
2069                "%s\n",
2070                acc_stats_border, acc_stats_border);
2071         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2072                "%-"PRIu64"\n"
2073                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2074                "%-"PRIu64"\n",
2075                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2076                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2077         if (total_rx_nombuf > 0)
2078                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2079         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
2080                "%s\n",
2081                acc_stats_border, acc_stats_border);
2082         if (record_core_cycles) {
2083 #define CYC_PER_MHZ 1E6
2084                 if (total_recv > 0 || total_xmit > 0) {
2085                         uint64_t total_pkts = 0;
2086                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2087                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2088                                 total_pkts = total_xmit;
2089                         else
2090                                 total_pkts = total_recv;
2091
2092                         printf("\n  CPU cycles/packet=%.2F (total cycles="
2093                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2094                                " MHz Clock\n",
2095                                (double) fwd_cycles / total_pkts,
2096                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2097                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2098                 }
2099         }
2100 }
2101
2102 void
2103 fwd_stats_reset(void)
2104 {
2105         streamid_t sm_id;
2106         portid_t pt_id;
2107         int i;
2108
2109         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2110                 pt_id = fwd_ports_ids[i];
2111                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2112         }
2113         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2114                 struct fwd_stream *fs = fwd_streams[sm_id];
2115
2116                 fs->rx_packets = 0;
2117                 fs->tx_packets = 0;
2118                 fs->fwd_dropped = 0;
2119                 fs->rx_bad_ip_csum = 0;
2120                 fs->rx_bad_l4_csum = 0;
2121                 fs->rx_bad_outer_l4_csum = 0;
2122                 fs->rx_bad_outer_ip_csum = 0;
2123
2124                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2125                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2126                 fs->core_cycles = 0;
2127         }
2128 }
2129
2130 static void
2131 flush_fwd_rx_queues(void)
2132 {
2133         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2134         portid_t  rxp;
2135         portid_t port_id;
2136         queueid_t rxq;
2137         uint16_t  nb_rx;
2138         uint16_t  i;
2139         uint8_t   j;
2140         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2141         uint64_t timer_period;
2142
2143         if (num_procs > 1) {
2144                 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2145                 return;
2146         }
2147
2148         /* convert to number of cycles */
2149         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2150
2151         for (j = 0; j < 2; j++) {
2152                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2153                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2154                                 port_id = fwd_ports_ids[rxp];
2155                                 /**
2156                                 * testpmd can stuck in the below do while loop
2157                                 * if rte_eth_rx_burst() always returns nonzero
2158                                 * packets. So timer is added to exit this loop
2159                                 * after 1sec timer expiry.
2160                                 */
2161                                 prev_tsc = rte_rdtsc();
2162                                 do {
2163                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2164                                                 pkts_burst, MAX_PKT_BURST);
2165                                         for (i = 0; i < nb_rx; i++)
2166                                                 rte_pktmbuf_free(pkts_burst[i]);
2167
2168                                         cur_tsc = rte_rdtsc();
2169                                         diff_tsc = cur_tsc - prev_tsc;
2170                                         timer_tsc += diff_tsc;
2171                                 } while ((nb_rx > 0) &&
2172                                         (timer_tsc < timer_period));
2173                                 timer_tsc = 0;
2174                         }
2175                 }
2176                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2177         }
2178 }
2179
2180 static void
2181 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2182 {
2183         struct fwd_stream **fsm;
2184         streamid_t nb_fs;
2185         streamid_t sm_id;
2186 #ifdef RTE_LIB_BITRATESTATS
2187         uint64_t tics_per_1sec;
2188         uint64_t tics_datum;
2189         uint64_t tics_current;
2190         uint16_t i, cnt_ports;
2191
2192         cnt_ports = nb_ports;
2193         tics_datum = rte_rdtsc();
2194         tics_per_1sec = rte_get_timer_hz();
2195 #endif
2196         fsm = &fwd_streams[fc->stream_idx];
2197         nb_fs = fc->stream_nb;
2198         do {
2199                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2200                         (*pkt_fwd)(fsm[sm_id]);
2201 #ifdef RTE_LIB_BITRATESTATS
2202                 if (bitrate_enabled != 0 &&
2203                                 bitrate_lcore_id == rte_lcore_id()) {
2204                         tics_current = rte_rdtsc();
2205                         if (tics_current - tics_datum >= tics_per_1sec) {
2206                                 /* Periodic bitrate calculation */
2207                                 for (i = 0; i < cnt_ports; i++)
2208                                         rte_stats_bitrate_calc(bitrate_data,
2209                                                 ports_ids[i]);
2210                                 tics_datum = tics_current;
2211                         }
2212                 }
2213 #endif
2214 #ifdef RTE_LIB_LATENCYSTATS
2215                 if (latencystats_enabled != 0 &&
2216                                 latencystats_lcore_id == rte_lcore_id())
2217                         rte_latencystats_update();
2218 #endif
2219
2220         } while (! fc->stopped);
2221 }
2222
2223 static int
2224 start_pkt_forward_on_core(void *fwd_arg)
2225 {
2226         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2227                              cur_fwd_config.fwd_eng->packet_fwd);
2228         return 0;
2229 }
2230
2231 /*
2232  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2233  * Used to start communication flows in network loopback test configurations.
2234  */
2235 static int
2236 run_one_txonly_burst_on_core(void *fwd_arg)
2237 {
2238         struct fwd_lcore *fwd_lc;
2239         struct fwd_lcore tmp_lcore;
2240
2241         fwd_lc = (struct fwd_lcore *) fwd_arg;
2242         tmp_lcore = *fwd_lc;
2243         tmp_lcore.stopped = 1;
2244         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2245         return 0;
2246 }
2247
2248 /*
2249  * Launch packet forwarding:
2250  *     - Setup per-port forwarding context.
2251  *     - launch logical cores with their forwarding configuration.
2252  */
2253 static void
2254 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2255 {
2256         unsigned int i;
2257         unsigned int lc_id;
2258         int diag;
2259
2260         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2261                 lc_id = fwd_lcores_cpuids[i];
2262                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2263                         fwd_lcores[i]->stopped = 0;
2264                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2265                                                      fwd_lcores[i], lc_id);
2266                         if (diag != 0)
2267                                 fprintf(stderr,
2268                                         "launch lcore %u failed - diag=%d\n",
2269                                         lc_id, diag);
2270                 }
2271         }
2272 }
2273
2274 /*
2275  * Launch packet forwarding configuration.
2276  */
2277 void
2278 start_packet_forwarding(int with_tx_first)
2279 {
2280         port_fwd_begin_t port_fwd_begin;
2281         port_fwd_end_t  port_fwd_end;
2282         unsigned int i;
2283
2284         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2285                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2286
2287         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2288                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2289
2290         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2291                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2292                 (!nb_rxq || !nb_txq))
2293                 rte_exit(EXIT_FAILURE,
2294                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2295                         cur_fwd_eng->fwd_mode_name);
2296
2297         if (all_ports_started() == 0) {
2298                 fprintf(stderr, "Not all ports were started\n");
2299                 return;
2300         }
2301         if (test_done == 0) {
2302                 fprintf(stderr, "Packet forwarding already started\n");
2303                 return;
2304         }
2305
2306         fwd_config_setup();
2307
2308         pkt_fwd_config_display(&cur_fwd_config);
2309         if (!pkt_fwd_shared_rxq_check())
2310                 return;
2311
2312         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2313         if (port_fwd_begin != NULL) {
2314                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2315                         if (port_fwd_begin(fwd_ports_ids[i])) {
2316                                 fprintf(stderr,
2317                                         "Packet forwarding is not ready\n");
2318                                 return;
2319                         }
2320                 }
2321         }
2322
2323         if (with_tx_first) {
2324                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2325                 if (port_fwd_begin != NULL) {
2326                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2327                                 if (port_fwd_begin(fwd_ports_ids[i])) {
2328                                         fprintf(stderr,
2329                                                 "Packet forwarding is not ready\n");
2330                                         return;
2331                                 }
2332                         }
2333                 }
2334         }
2335
2336         test_done = 0;
2337
2338         if(!no_flush_rx)
2339                 flush_fwd_rx_queues();
2340
2341         rxtx_config_display();
2342
2343         fwd_stats_reset();
2344         if (with_tx_first) {
2345                 while (with_tx_first--) {
2346                         launch_packet_forwarding(
2347                                         run_one_txonly_burst_on_core);
2348                         rte_eal_mp_wait_lcore();
2349                 }
2350                 port_fwd_end = tx_only_engine.port_fwd_end;
2351                 if (port_fwd_end != NULL) {
2352                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2353                                 (*port_fwd_end)(fwd_ports_ids[i]);
2354                 }
2355         }
2356         launch_packet_forwarding(start_pkt_forward_on_core);
2357 }
2358
2359 void
2360 stop_packet_forwarding(void)
2361 {
2362         port_fwd_end_t port_fwd_end;
2363         lcoreid_t lc_id;
2364         portid_t pt_id;
2365         int i;
2366
2367         if (test_done) {
2368                 fprintf(stderr, "Packet forwarding not started\n");
2369                 return;
2370         }
2371         printf("Telling cores to stop...");
2372         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2373                 fwd_lcores[lc_id]->stopped = 1;
2374         printf("\nWaiting for lcores to finish...\n");
2375         rte_eal_mp_wait_lcore();
2376         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2377         if (port_fwd_end != NULL) {
2378                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2379                         pt_id = fwd_ports_ids[i];
2380                         (*port_fwd_end)(pt_id);
2381                 }
2382         }
2383
2384         fwd_stats_display();
2385
2386         printf("\nDone.\n");
2387         test_done = 1;
2388 }
2389
2390 void
2391 dev_set_link_up(portid_t pid)
2392 {
2393         if (rte_eth_dev_set_link_up(pid) < 0)
2394                 fprintf(stderr, "\nSet link up fail.\n");
2395 }
2396
2397 void
2398 dev_set_link_down(portid_t pid)
2399 {
2400         if (rte_eth_dev_set_link_down(pid) < 0)
2401                 fprintf(stderr, "\nSet link down fail.\n");
2402 }
2403
2404 static int
2405 all_ports_started(void)
2406 {
2407         portid_t pi;
2408         struct rte_port *port;
2409
2410         RTE_ETH_FOREACH_DEV(pi) {
2411                 port = &ports[pi];
2412                 /* Check if there is a port which is not started */
2413                 if ((port->port_status != RTE_PORT_STARTED) &&
2414                         (port->slave_flag == 0))
2415                         return 0;
2416         }
2417
2418         /* No port is not started */
2419         return 1;
2420 }
2421
2422 int
2423 port_is_stopped(portid_t port_id)
2424 {
2425         struct rte_port *port = &ports[port_id];
2426
2427         if ((port->port_status != RTE_PORT_STOPPED) &&
2428             (port->slave_flag == 0))
2429                 return 0;
2430         return 1;
2431 }
2432
2433 int
2434 all_ports_stopped(void)
2435 {
2436         portid_t pi;
2437
2438         RTE_ETH_FOREACH_DEV(pi) {
2439                 if (!port_is_stopped(pi))
2440                         return 0;
2441         }
2442
2443         return 1;
2444 }
2445
2446 int
2447 port_is_started(portid_t port_id)
2448 {
2449         if (port_id_is_invalid(port_id, ENABLED_WARN))
2450                 return 0;
2451
2452         if (ports[port_id].port_status != RTE_PORT_STARTED)
2453                 return 0;
2454
2455         return 1;
2456 }
2457
2458 /* Configure the Rx and Tx hairpin queues for the selected port. */
2459 static int
2460 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2461 {
2462         queueid_t qi;
2463         struct rte_eth_hairpin_conf hairpin_conf = {
2464                 .peer_count = 1,
2465         };
2466         int i;
2467         int diag;
2468         struct rte_port *port = &ports[pi];
2469         uint16_t peer_rx_port = pi;
2470         uint16_t peer_tx_port = pi;
2471         uint32_t manual = 1;
2472         uint32_t tx_exp = hairpin_mode & 0x10;
2473
2474         if (!(hairpin_mode & 0xf)) {
2475                 peer_rx_port = pi;
2476                 peer_tx_port = pi;
2477                 manual = 0;
2478         } else if (hairpin_mode & 0x1) {
2479                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2480                                                        RTE_ETH_DEV_NO_OWNER);
2481                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2482                         peer_tx_port = rte_eth_find_next_owned_by(0,
2483                                                 RTE_ETH_DEV_NO_OWNER);
2484                 if (p_pi != RTE_MAX_ETHPORTS) {
2485                         peer_rx_port = p_pi;
2486                 } else {
2487                         uint16_t next_pi;
2488
2489                         /* Last port will be the peer RX port of the first. */
2490                         RTE_ETH_FOREACH_DEV(next_pi)
2491                                 peer_rx_port = next_pi;
2492                 }
2493                 manual = 1;
2494         } else if (hairpin_mode & 0x2) {
2495                 if (cnt_pi & 0x1) {
2496                         peer_rx_port = p_pi;
2497                 } else {
2498                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2499                                                 RTE_ETH_DEV_NO_OWNER);
2500                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2501                                 peer_rx_port = pi;
2502                 }
2503                 peer_tx_port = peer_rx_port;
2504                 manual = 1;
2505         }
2506
2507         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2508                 hairpin_conf.peers[0].port = peer_rx_port;
2509                 hairpin_conf.peers[0].queue = i + nb_rxq;
2510                 hairpin_conf.manual_bind = !!manual;
2511                 hairpin_conf.tx_explicit = !!tx_exp;
2512                 diag = rte_eth_tx_hairpin_queue_setup
2513                         (pi, qi, nb_txd, &hairpin_conf);
2514                 i++;
2515                 if (diag == 0)
2516                         continue;
2517
2518                 /* Fail to setup rx queue, return */
2519                 if (port->port_status == RTE_PORT_HANDLING)
2520                         port->port_status = RTE_PORT_STOPPED;
2521                 else
2522                         fprintf(stderr,
2523                                 "Port %d can not be set back to stopped\n", pi);
2524                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2525                         pi);
2526                 /* try to reconfigure queues next time */
2527                 port->need_reconfig_queues = 1;
2528                 return -1;
2529         }
2530         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2531                 hairpin_conf.peers[0].port = peer_tx_port;
2532                 hairpin_conf.peers[0].queue = i + nb_txq;
2533                 hairpin_conf.manual_bind = !!manual;
2534                 hairpin_conf.tx_explicit = !!tx_exp;
2535                 diag = rte_eth_rx_hairpin_queue_setup
2536                         (pi, qi, nb_rxd, &hairpin_conf);
2537                 i++;
2538                 if (diag == 0)
2539                         continue;
2540
2541                 /* Fail to setup rx queue, return */
2542                 if (port->port_status == RTE_PORT_HANDLING)
2543                         port->port_status = RTE_PORT_STOPPED;
2544                 else
2545                         fprintf(stderr,
2546                                 "Port %d can not be set back to stopped\n", pi);
2547                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2548                         pi);
2549                 /* try to reconfigure queues next time */
2550                 port->need_reconfig_queues = 1;
2551                 return -1;
2552         }
2553         return 0;
2554 }
2555
2556 /* Configure the Rx with optional split. */
2557 int
2558 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2559                uint16_t nb_rx_desc, unsigned int socket_id,
2560                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2561 {
2562         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2563         unsigned int i, mp_n;
2564         int ret;
2565
2566         if (rx_pkt_nb_segs <= 1 ||
2567             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2568                 rx_conf->rx_seg = NULL;
2569                 rx_conf->rx_nseg = 0;
2570                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2571                                              nb_rx_desc, socket_id,
2572                                              rx_conf, mp);
2573                 return ret;
2574         }
2575         for (i = 0; i < rx_pkt_nb_segs; i++) {
2576                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2577                 struct rte_mempool *mpx;
2578                 /*
2579                  * Use last valid pool for the segments with number
2580                  * exceeding the pool index.
2581                  */
2582                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2583                 mpx = mbuf_pool_find(socket_id, mp_n);
2584                 /* Handle zero as mbuf data buffer size. */
2585                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2586                                    rx_pkt_seg_lengths[i] :
2587                                    mbuf_data_size[mp_n];
2588                 rx_seg->offset = i < rx_pkt_nb_offs ?
2589                                    rx_pkt_seg_offsets[i] : 0;
2590                 rx_seg->mp = mpx ? mpx : mp;
2591         }
2592         rx_conf->rx_nseg = rx_pkt_nb_segs;
2593         rx_conf->rx_seg = rx_useg;
2594         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2595                                     socket_id, rx_conf, NULL);
2596         rx_conf->rx_seg = NULL;
2597         rx_conf->rx_nseg = 0;
2598         return ret;
2599 }
2600
2601 static int
2602 alloc_xstats_display_info(portid_t pi)
2603 {
2604         uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2605         uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2606         uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2607
2608         if (xstats_display_num == 0)
2609                 return 0;
2610
2611         *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2612         if (*ids_supp == NULL)
2613                 goto fail_ids_supp;
2614
2615         *prev_values = calloc(xstats_display_num,
2616                               sizeof(**prev_values));
2617         if (*prev_values == NULL)
2618                 goto fail_prev_values;
2619
2620         *curr_values = calloc(xstats_display_num,
2621                               sizeof(**curr_values));
2622         if (*curr_values == NULL)
2623                 goto fail_curr_values;
2624
2625         ports[pi].xstats_info.allocated = true;
2626
2627         return 0;
2628
2629 fail_curr_values:
2630         free(*prev_values);
2631 fail_prev_values:
2632         free(*ids_supp);
2633 fail_ids_supp:
2634         return -ENOMEM;
2635 }
2636
2637 static void
2638 free_xstats_display_info(portid_t pi)
2639 {
2640         if (!ports[pi].xstats_info.allocated)
2641                 return;
2642         free(ports[pi].xstats_info.ids_supp);
2643         free(ports[pi].xstats_info.prev_values);
2644         free(ports[pi].xstats_info.curr_values);
2645         ports[pi].xstats_info.allocated = false;
2646 }
2647
2648 /** Fill helper structures for specified port to show extended statistics. */
2649 static void
2650 fill_xstats_display_info_for_port(portid_t pi)
2651 {
2652         unsigned int stat, stat_supp;
2653         const char *xstat_name;
2654         struct rte_port *port;
2655         uint64_t *ids_supp;
2656         int rc;
2657
2658         if (xstats_display_num == 0)
2659                 return;
2660
2661         if (pi == (portid_t)RTE_PORT_ALL) {
2662                 fill_xstats_display_info();
2663                 return;
2664         }
2665
2666         port = &ports[pi];
2667         if (port->port_status != RTE_PORT_STARTED)
2668                 return;
2669
2670         if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2671                 rte_exit(EXIT_FAILURE,
2672                          "Failed to allocate xstats display memory\n");
2673
2674         ids_supp = port->xstats_info.ids_supp;
2675         for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2676                 xstat_name = xstats_display[stat].name;
2677                 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2678                                                    ids_supp + stat_supp);
2679                 if (rc != 0) {
2680                         fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2681                                 xstat_name, pi, stat);
2682                         continue;
2683                 }
2684                 stat_supp++;
2685         }
2686
2687         port->xstats_info.ids_supp_sz = stat_supp;
2688 }
2689
2690 /** Fill helper structures for all ports to show extended statistics. */
2691 static void
2692 fill_xstats_display_info(void)
2693 {
2694         portid_t pi;
2695
2696         if (xstats_display_num == 0)
2697                 return;
2698
2699         RTE_ETH_FOREACH_DEV(pi)
2700                 fill_xstats_display_info_for_port(pi);
2701 }
2702
2703 int
2704 start_port(portid_t pid)
2705 {
2706         int diag, need_check_link_status = -1;
2707         portid_t pi;
2708         portid_t p_pi = RTE_MAX_ETHPORTS;
2709         portid_t pl[RTE_MAX_ETHPORTS];
2710         portid_t peer_pl[RTE_MAX_ETHPORTS];
2711         uint16_t cnt_pi = 0;
2712         uint16_t cfg_pi = 0;
2713         int peer_pi;
2714         queueid_t qi;
2715         struct rte_port *port;
2716         struct rte_eth_hairpin_cap cap;
2717
2718         if (port_id_is_invalid(pid, ENABLED_WARN))
2719                 return 0;
2720
2721         RTE_ETH_FOREACH_DEV(pi) {
2722                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2723                         continue;
2724
2725                 need_check_link_status = 0;
2726                 port = &ports[pi];
2727                 if (port->port_status == RTE_PORT_STOPPED)
2728                         port->port_status = RTE_PORT_HANDLING;
2729                 else {
2730                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2731                         continue;
2732                 }
2733
2734                 if (port->need_reconfig > 0) {
2735                         struct rte_eth_conf dev_conf;
2736                         int k;
2737
2738                         port->need_reconfig = 0;
2739
2740                         if (flow_isolate_all) {
2741                                 int ret = port_flow_isolate(pi, 1);
2742                                 if (ret) {
2743                                         fprintf(stderr,
2744                                                 "Failed to apply isolated mode on port %d\n",
2745                                                 pi);
2746                                         return -1;
2747                                 }
2748                         }
2749                         configure_rxtx_dump_callbacks(0);
2750                         printf("Configuring Port %d (socket %u)\n", pi,
2751                                         port->socket_id);
2752                         if (nb_hairpinq > 0 &&
2753                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2754                                 fprintf(stderr,
2755                                         "Port %d doesn't support hairpin queues\n",
2756                                         pi);
2757                                 return -1;
2758                         }
2759
2760                         /* configure port */
2761                         diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2762                                                      nb_txq + nb_hairpinq,
2763                                                      &(port->dev_conf));
2764                         if (diag != 0) {
2765                                 if (port->port_status == RTE_PORT_HANDLING)
2766                                         port->port_status = RTE_PORT_STOPPED;
2767                                 else
2768                                         fprintf(stderr,
2769                                                 "Port %d can not be set back to stopped\n",
2770                                                 pi);
2771                                 fprintf(stderr, "Fail to configure port %d\n",
2772                                         pi);
2773                                 /* try to reconfigure port next time */
2774                                 port->need_reconfig = 1;
2775                                 return -1;
2776                         }
2777                         /* get device configuration*/
2778                         if (0 !=
2779                                 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2780                                 fprintf(stderr,
2781                                         "port %d can not get device configuration\n",
2782                                         pi);
2783                                 return -1;
2784                         }
2785                         /* Apply Rx offloads configuration */
2786                         if (dev_conf.rxmode.offloads !=
2787                             port->dev_conf.rxmode.offloads) {
2788                                 port->dev_conf.rxmode.offloads |=
2789                                         dev_conf.rxmode.offloads;
2790                                 for (k = 0;
2791                                      k < port->dev_info.max_rx_queues;
2792                                      k++)
2793                                         port->rx_conf[k].offloads |=
2794                                                 dev_conf.rxmode.offloads;
2795                         }
2796                         /* Apply Tx offloads configuration */
2797                         if (dev_conf.txmode.offloads !=
2798                             port->dev_conf.txmode.offloads) {
2799                                 port->dev_conf.txmode.offloads |=
2800                                         dev_conf.txmode.offloads;
2801                                 for (k = 0;
2802                                      k < port->dev_info.max_tx_queues;
2803                                      k++)
2804                                         port->tx_conf[k].offloads |=
2805                                                 dev_conf.txmode.offloads;
2806                         }
2807                 }
2808                 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2809                         port->need_reconfig_queues = 0;
2810                         /* setup tx queues */
2811                         for (qi = 0; qi < nb_txq; qi++) {
2812                                 if ((numa_support) &&
2813                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2814                                         diag = rte_eth_tx_queue_setup(pi, qi,
2815                                                 port->nb_tx_desc[qi],
2816                                                 txring_numa[pi],
2817                                                 &(port->tx_conf[qi]));
2818                                 else
2819                                         diag = rte_eth_tx_queue_setup(pi, qi,
2820                                                 port->nb_tx_desc[qi],
2821                                                 port->socket_id,
2822                                                 &(port->tx_conf[qi]));
2823
2824                                 if (diag == 0)
2825                                         continue;
2826
2827                                 /* Fail to setup tx queue, return */
2828                                 if (port->port_status == RTE_PORT_HANDLING)
2829                                         port->port_status = RTE_PORT_STOPPED;
2830                                 else
2831                                         fprintf(stderr,
2832                                                 "Port %d can not be set back to stopped\n",
2833                                                 pi);
2834                                 fprintf(stderr,
2835                                         "Fail to configure port %d tx queues\n",
2836                                         pi);
2837                                 /* try to reconfigure queues next time */
2838                                 port->need_reconfig_queues = 1;
2839                                 return -1;
2840                         }
2841                         for (qi = 0; qi < nb_rxq; qi++) {
2842                                 /* setup rx queues */
2843                                 if ((numa_support) &&
2844                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2845                                         struct rte_mempool * mp =
2846                                                 mbuf_pool_find
2847                                                         (rxring_numa[pi], 0);
2848                                         if (mp == NULL) {
2849                                                 fprintf(stderr,
2850                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2851                                                         rxring_numa[pi]);
2852                                                 return -1;
2853                                         }
2854
2855                                         diag = rx_queue_setup(pi, qi,
2856                                              port->nb_rx_desc[qi],
2857                                              rxring_numa[pi],
2858                                              &(port->rx_conf[qi]),
2859                                              mp);
2860                                 } else {
2861                                         struct rte_mempool *mp =
2862                                                 mbuf_pool_find
2863                                                         (port->socket_id, 0);
2864                                         if (mp == NULL) {
2865                                                 fprintf(stderr,
2866                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2867                                                         port->socket_id);
2868                                                 return -1;
2869                                         }
2870                                         diag = rx_queue_setup(pi, qi,
2871                                              port->nb_rx_desc[qi],
2872                                              port->socket_id,
2873                                              &(port->rx_conf[qi]),
2874                                              mp);
2875                                 }
2876                                 if (diag == 0)
2877                                         continue;
2878
2879                                 /* Fail to setup rx queue, return */
2880                                 if (port->port_status == RTE_PORT_HANDLING)
2881                                         port->port_status = RTE_PORT_STOPPED;
2882                                 else
2883                                         fprintf(stderr,
2884                                                 "Port %d can not be set back to stopped\n",
2885                                                 pi);
2886                                 fprintf(stderr,
2887                                         "Fail to configure port %d rx queues\n",
2888                                         pi);
2889                                 /* try to reconfigure queues next time */
2890                                 port->need_reconfig_queues = 1;
2891                                 return -1;
2892                         }
2893                         /* setup hairpin queues */
2894                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2895                                 return -1;
2896                 }
2897                 configure_rxtx_dump_callbacks(verbose_level);
2898                 if (clear_ptypes) {
2899                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2900                                         NULL, 0);
2901                         if (diag < 0)
2902                                 fprintf(stderr,
2903                                         "Port %d: Failed to disable Ptype parsing\n",
2904                                         pi);
2905                 }
2906
2907                 p_pi = pi;
2908                 cnt_pi++;
2909
2910                 /* start port */
2911                 diag = eth_dev_start_mp(pi);
2912                 if (diag < 0) {
2913                         fprintf(stderr, "Fail to start port %d: %s\n",
2914                                 pi, rte_strerror(-diag));
2915
2916                         /* Fail to setup rx queue, return */
2917                         if (port->port_status == RTE_PORT_HANDLING)
2918                                 port->port_status = RTE_PORT_STOPPED;
2919                         else
2920                                 fprintf(stderr,
2921                                         "Port %d can not be set back to stopped\n",
2922                                         pi);
2923                         continue;
2924                 }
2925
2926                 if (port->port_status == RTE_PORT_HANDLING)
2927                         port->port_status = RTE_PORT_STARTED;
2928                 else
2929                         fprintf(stderr, "Port %d can not be set into started\n",
2930                                 pi);
2931
2932                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
2933                         printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
2934                                         RTE_ETHER_ADDR_BYTES(&port->eth_addr));
2935
2936                 /* at least one port started, need checking link status */
2937                 need_check_link_status = 1;
2938
2939                 pl[cfg_pi++] = pi;
2940         }
2941
2942         if (need_check_link_status == 1 && !no_link_check)
2943                 check_all_ports_link_status(RTE_PORT_ALL);
2944         else if (need_check_link_status == 0)
2945                 fprintf(stderr, "Please stop the ports first\n");
2946
2947         if (hairpin_mode & 0xf) {
2948                 uint16_t i;
2949                 int j;
2950
2951                 /* bind all started hairpin ports */
2952                 for (i = 0; i < cfg_pi; i++) {
2953                         pi = pl[i];
2954                         /* bind current Tx to all peer Rx */
2955                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2956                                                         RTE_MAX_ETHPORTS, 1);
2957                         if (peer_pi < 0)
2958                                 return peer_pi;
2959                         for (j = 0; j < peer_pi; j++) {
2960                                 if (!port_is_started(peer_pl[j]))
2961                                         continue;
2962                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2963                                 if (diag < 0) {
2964                                         fprintf(stderr,
2965                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
2966                                                 pi, peer_pl[j],
2967                                                 rte_strerror(-diag));
2968                                         return -1;
2969                                 }
2970                         }
2971                         /* bind all peer Tx to current Rx */
2972                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2973                                                         RTE_MAX_ETHPORTS, 0);
2974                         if (peer_pi < 0)
2975                                 return peer_pi;
2976                         for (j = 0; j < peer_pi; j++) {
2977                                 if (!port_is_started(peer_pl[j]))
2978                                         continue;
2979                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2980                                 if (diag < 0) {
2981                                         fprintf(stderr,
2982                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
2983                                                 peer_pl[j], pi,
2984                                                 rte_strerror(-diag));
2985                                         return -1;
2986                                 }
2987                         }
2988                 }
2989         }
2990
2991         fill_xstats_display_info_for_port(pid);
2992
2993         printf("Done\n");
2994         return 0;
2995 }
2996
2997 void
2998 stop_port(portid_t pid)
2999 {
3000         portid_t pi;
3001         struct rte_port *port;
3002         int need_check_link_status = 0;
3003         portid_t peer_pl[RTE_MAX_ETHPORTS];
3004         int peer_pi;
3005
3006         if (port_id_is_invalid(pid, ENABLED_WARN))
3007                 return;
3008
3009         printf("Stopping ports...\n");
3010
3011         RTE_ETH_FOREACH_DEV(pi) {
3012                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3013                         continue;
3014
3015                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3016                         fprintf(stderr,
3017                                 "Please remove port %d from forwarding configuration.\n",
3018                                 pi);
3019                         continue;
3020                 }
3021
3022                 if (port_is_bonding_slave(pi)) {
3023                         fprintf(stderr,
3024                                 "Please remove port %d from bonded device.\n",
3025                                 pi);
3026                         continue;
3027                 }
3028
3029                 port = &ports[pi];
3030                 if (port->port_status == RTE_PORT_STARTED)
3031                         port->port_status = RTE_PORT_HANDLING;
3032                 else
3033                         continue;
3034
3035                 if (hairpin_mode & 0xf) {
3036                         int j;
3037
3038                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3039                         /* unbind all peer Tx from current Rx */
3040                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3041                                                         RTE_MAX_ETHPORTS, 0);
3042                         if (peer_pi < 0)
3043                                 continue;
3044                         for (j = 0; j < peer_pi; j++) {
3045                                 if (!port_is_started(peer_pl[j]))
3046                                         continue;
3047                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
3048                         }
3049                 }
3050
3051                 if (port->flow_list)
3052                         port_flow_flush(pi);
3053
3054                 if (eth_dev_stop_mp(pi) != 0)
3055                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3056                                 pi);
3057
3058                 if (port->port_status == RTE_PORT_HANDLING)
3059                         port->port_status = RTE_PORT_STOPPED;
3060                 else
3061                         fprintf(stderr, "Port %d can not be set into stopped\n",
3062                                 pi);
3063                 need_check_link_status = 1;
3064         }
3065         if (need_check_link_status && !no_link_check)
3066                 check_all_ports_link_status(RTE_PORT_ALL);
3067
3068         printf("Done\n");
3069 }
3070
3071 static void
3072 remove_invalid_ports_in(portid_t *array, portid_t *total)
3073 {
3074         portid_t i;
3075         portid_t new_total = 0;
3076
3077         for (i = 0; i < *total; i++)
3078                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3079                         array[new_total] = array[i];
3080                         new_total++;
3081                 }
3082         *total = new_total;
3083 }
3084
3085 static void
3086 remove_invalid_ports(void)
3087 {
3088         remove_invalid_ports_in(ports_ids, &nb_ports);
3089         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3090         nb_cfg_ports = nb_fwd_ports;
3091 }
3092
3093 void
3094 close_port(portid_t pid)
3095 {
3096         portid_t pi;
3097         struct rte_port *port;
3098
3099         if (port_id_is_invalid(pid, ENABLED_WARN))
3100                 return;
3101
3102         printf("Closing ports...\n");
3103
3104         RTE_ETH_FOREACH_DEV(pi) {
3105                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3106                         continue;
3107
3108                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3109                         fprintf(stderr,
3110                                 "Please remove port %d from forwarding configuration.\n",
3111                                 pi);
3112                         continue;
3113                 }
3114
3115                 if (port_is_bonding_slave(pi)) {
3116                         fprintf(stderr,
3117                                 "Please remove port %d from bonded device.\n",
3118                                 pi);
3119                         continue;
3120                 }
3121
3122                 port = &ports[pi];
3123                 if (port->port_status == RTE_PORT_CLOSED) {
3124                         fprintf(stderr, "Port %d is already closed\n", pi);
3125                         continue;
3126                 }
3127
3128                 if (is_proc_primary()) {
3129                         port_flow_flush(pi);
3130                         port_flex_item_flush(pi);
3131                         rte_eth_dev_close(pi);
3132                 }
3133
3134                 free_xstats_display_info(pi);
3135         }
3136
3137         remove_invalid_ports();
3138         printf("Done\n");
3139 }
3140
3141 void
3142 reset_port(portid_t pid)
3143 {
3144         int diag;
3145         portid_t pi;
3146         struct rte_port *port;
3147
3148         if (port_id_is_invalid(pid, ENABLED_WARN))
3149                 return;
3150
3151         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3152                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3153                 fprintf(stderr,
3154                         "Can not reset port(s), please stop port(s) first.\n");
3155                 return;
3156         }
3157
3158         printf("Resetting ports...\n");
3159
3160         RTE_ETH_FOREACH_DEV(pi) {
3161                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3162                         continue;
3163
3164                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3165                         fprintf(stderr,
3166                                 "Please remove port %d from forwarding configuration.\n",
3167                                 pi);
3168                         continue;
3169                 }
3170
3171                 if (port_is_bonding_slave(pi)) {
3172                         fprintf(stderr,
3173                                 "Please remove port %d from bonded device.\n",
3174                                 pi);
3175                         continue;
3176                 }
3177
3178                 diag = rte_eth_dev_reset(pi);
3179                 if (diag == 0) {
3180                         port = &ports[pi];
3181                         port->need_reconfig = 1;
3182                         port->need_reconfig_queues = 1;
3183                 } else {
3184                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3185                                 pi, diag);
3186                 }
3187         }
3188
3189         printf("Done\n");
3190 }
3191
3192 void
3193 attach_port(char *identifier)
3194 {
3195         portid_t pi;
3196         struct rte_dev_iterator iterator;
3197
3198         printf("Attaching a new port...\n");
3199
3200         if (identifier == NULL) {
3201                 fprintf(stderr, "Invalid parameters are specified\n");
3202                 return;
3203         }
3204
3205         if (rte_dev_probe(identifier) < 0) {
3206                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3207                 return;
3208         }
3209
3210         /* first attach mode: event */
3211         if (setup_on_probe_event) {
3212                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
3213                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3214                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
3215                                         ports[pi].need_setup != 0)
3216                                 setup_attached_port(pi);
3217                 return;
3218         }
3219
3220         /* second attach mode: iterator */
3221         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3222                 /* setup ports matching the devargs used for probing */
3223                 if (port_is_forwarding(pi))
3224                         continue; /* port was already attached before */
3225                 setup_attached_port(pi);
3226         }
3227 }
3228
3229 static void
3230 setup_attached_port(portid_t pi)
3231 {
3232         unsigned int socket_id;
3233         int ret;
3234
3235         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3236         /* if socket_id is invalid, set to the first available socket. */
3237         if (check_socket_id(socket_id) < 0)
3238                 socket_id = socket_ids[0];
3239         reconfig(pi, socket_id);
3240         ret = rte_eth_promiscuous_enable(pi);
3241         if (ret != 0)
3242                 fprintf(stderr,
3243                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3244                         pi, rte_strerror(-ret));
3245
3246         ports_ids[nb_ports++] = pi;
3247         fwd_ports_ids[nb_fwd_ports++] = pi;
3248         nb_cfg_ports = nb_fwd_ports;
3249         ports[pi].need_setup = 0;
3250         ports[pi].port_status = RTE_PORT_STOPPED;
3251
3252         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3253         printf("Done\n");
3254 }
3255
3256 static void
3257 detach_device(struct rte_device *dev)
3258 {
3259         portid_t sibling;
3260
3261         if (dev == NULL) {
3262                 fprintf(stderr, "Device already removed\n");
3263                 return;
3264         }
3265
3266         printf("Removing a device...\n");
3267
3268         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3269                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3270                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3271                                 fprintf(stderr, "Port %u not stopped\n",
3272                                         sibling);
3273                                 return;
3274                         }
3275                         port_flow_flush(sibling);
3276                 }
3277         }
3278
3279         if (rte_dev_remove(dev) < 0) {
3280                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3281                 return;
3282         }
3283         remove_invalid_ports();
3284
3285         printf("Device is detached\n");
3286         printf("Now total ports is %d\n", nb_ports);
3287         printf("Done\n");
3288         return;
3289 }
3290
3291 void
3292 detach_port_device(portid_t port_id)
3293 {
3294         int ret;
3295         struct rte_eth_dev_info dev_info;
3296
3297         if (port_id_is_invalid(port_id, ENABLED_WARN))
3298                 return;
3299
3300         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3301                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3302                         fprintf(stderr, "Port not stopped\n");
3303                         return;
3304                 }
3305                 fprintf(stderr, "Port was not closed\n");
3306         }
3307
3308         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3309         if (ret != 0) {
3310                 TESTPMD_LOG(ERR,
3311                         "Failed to get device info for port %d, not detaching\n",
3312                         port_id);
3313                 return;
3314         }
3315         detach_device(dev_info.device);
3316 }
3317
3318 void
3319 detach_devargs(char *identifier)
3320 {
3321         struct rte_dev_iterator iterator;
3322         struct rte_devargs da;
3323         portid_t port_id;
3324
3325         printf("Removing a device...\n");
3326
3327         memset(&da, 0, sizeof(da));
3328         if (rte_devargs_parsef(&da, "%s", identifier)) {
3329                 fprintf(stderr, "cannot parse identifier\n");
3330                 return;
3331         }
3332
3333         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3334                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3335                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3336                                 fprintf(stderr, "Port %u not stopped\n",
3337                                         port_id);
3338                                 rte_eth_iterator_cleanup(&iterator);
3339                                 rte_devargs_reset(&da);
3340                                 return;
3341                         }
3342                         port_flow_flush(port_id);
3343                 }
3344         }
3345
3346         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3347                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3348                             da.name, da.bus->name);
3349                 rte_devargs_reset(&da);
3350                 return;
3351         }
3352
3353         remove_invalid_ports();
3354
3355         printf("Device %s is detached\n", identifier);
3356         printf("Now total ports is %d\n", nb_ports);
3357         printf("Done\n");
3358         rte_devargs_reset(&da);
3359 }
3360
3361 void
3362 pmd_test_exit(void)
3363 {
3364         portid_t pt_id;
3365         unsigned int i;
3366         int ret;
3367
3368         if (test_done == 0)
3369                 stop_packet_forwarding();
3370
3371 #ifndef RTE_EXEC_ENV_WINDOWS
3372         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3373                 if (mempools[i]) {
3374                         if (mp_alloc_type == MP_ALLOC_ANON)
3375                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3376                                                      NULL);
3377                 }
3378         }
3379 #endif
3380         if (ports != NULL) {
3381                 no_link_check = 1;
3382                 RTE_ETH_FOREACH_DEV(pt_id) {
3383                         printf("\nStopping port %d...\n", pt_id);
3384                         fflush(stdout);
3385                         stop_port(pt_id);
3386                 }
3387                 RTE_ETH_FOREACH_DEV(pt_id) {
3388                         printf("\nShutting down port %d...\n", pt_id);
3389                         fflush(stdout);
3390                         close_port(pt_id);
3391                 }
3392         }
3393
3394         if (hot_plug) {
3395                 ret = rte_dev_event_monitor_stop();
3396                 if (ret) {
3397                         RTE_LOG(ERR, EAL,
3398                                 "fail to stop device event monitor.");
3399                         return;
3400                 }
3401
3402                 ret = rte_dev_event_callback_unregister(NULL,
3403                         dev_event_callback, NULL);
3404                 if (ret < 0) {
3405                         RTE_LOG(ERR, EAL,
3406                                 "fail to unregister device event callback.\n");
3407                         return;
3408                 }
3409
3410                 ret = rte_dev_hotplug_handle_disable();
3411                 if (ret) {
3412                         RTE_LOG(ERR, EAL,
3413                                 "fail to disable hotplug handling.\n");
3414                         return;
3415                 }
3416         }
3417         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3418                 if (mempools[i])
3419                         mempool_free_mp(mempools[i]);
3420         }
3421         free(xstats_display);
3422
3423         printf("\nBye...\n");
3424 }
3425
3426 typedef void (*cmd_func_t)(void);
3427 struct pmd_test_command {
3428         const char *cmd_name;
3429         cmd_func_t cmd_func;
3430 };
3431
3432 /* Check the link status of all ports in up to 9s, and print them finally */
3433 static void
3434 check_all_ports_link_status(uint32_t port_mask)
3435 {
3436 #define CHECK_INTERVAL 100 /* 100ms */
3437 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3438         portid_t portid;
3439         uint8_t count, all_ports_up, print_flag = 0;
3440         struct rte_eth_link link;
3441         int ret;
3442         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3443
3444         printf("Checking link statuses...\n");
3445         fflush(stdout);
3446         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3447                 all_ports_up = 1;
3448                 RTE_ETH_FOREACH_DEV(portid) {
3449                         if ((port_mask & (1 << portid)) == 0)
3450                                 continue;
3451                         memset(&link, 0, sizeof(link));
3452                         ret = rte_eth_link_get_nowait(portid, &link);
3453                         if (ret < 0) {
3454                                 all_ports_up = 0;
3455                                 if (print_flag == 1)
3456                                         fprintf(stderr,
3457                                                 "Port %u link get failed: %s\n",
3458                                                 portid, rte_strerror(-ret));
3459                                 continue;
3460                         }
3461                         /* print link status if flag set */
3462                         if (print_flag == 1) {
3463                                 rte_eth_link_to_str(link_status,
3464                                         sizeof(link_status), &link);
3465                                 printf("Port %d %s\n", portid, link_status);
3466                                 continue;
3467                         }
3468                         /* clear all_ports_up flag if any link down */
3469                         if (link.link_status == RTE_ETH_LINK_DOWN) {
3470                                 all_ports_up = 0;
3471                                 break;
3472                         }
3473                 }
3474                 /* after finally printing all link status, get out */
3475                 if (print_flag == 1)
3476                         break;
3477
3478                 if (all_ports_up == 0) {
3479                         fflush(stdout);
3480                         rte_delay_ms(CHECK_INTERVAL);
3481                 }
3482
3483                 /* set the print_flag if all ports up or timeout */
3484                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3485                         print_flag = 1;
3486                 }
3487
3488                 if (lsc_interrupt)
3489                         break;
3490         }
3491 }
3492
3493 static void
3494 rmv_port_callback(void *arg)
3495 {
3496         int need_to_start = 0;
3497         int org_no_link_check = no_link_check;
3498         portid_t port_id = (intptr_t)arg;
3499         struct rte_eth_dev_info dev_info;
3500         int ret;
3501
3502         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3503
3504         if (!test_done && port_is_forwarding(port_id)) {
3505                 need_to_start = 1;
3506                 stop_packet_forwarding();
3507         }
3508         no_link_check = 1;
3509         stop_port(port_id);
3510         no_link_check = org_no_link_check;
3511
3512         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3513         if (ret != 0)
3514                 TESTPMD_LOG(ERR,
3515                         "Failed to get device info for port %d, not detaching\n",
3516                         port_id);
3517         else {
3518                 struct rte_device *device = dev_info.device;
3519                 close_port(port_id);
3520                 detach_device(device); /* might be already removed or have more ports */
3521         }
3522         if (need_to_start)
3523                 start_packet_forwarding(0);
3524 }
3525
3526 /* This function is used by the interrupt thread */
3527 static int
3528 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3529                   void *ret_param)
3530 {
3531         RTE_SET_USED(param);
3532         RTE_SET_USED(ret_param);
3533
3534         if (type >= RTE_ETH_EVENT_MAX) {
3535                 fprintf(stderr,
3536                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3537                         port_id, __func__, type);
3538                 fflush(stderr);
3539         } else if (event_print_mask & (UINT32_C(1) << type)) {
3540                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3541                         eth_event_desc[type]);
3542                 fflush(stdout);
3543         }
3544
3545         switch (type) {
3546         case RTE_ETH_EVENT_NEW:
3547                 ports[port_id].need_setup = 1;
3548                 ports[port_id].port_status = RTE_PORT_HANDLING;
3549                 break;
3550         case RTE_ETH_EVENT_INTR_RMV:
3551                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3552                         break;
3553                 if (rte_eal_alarm_set(100000,
3554                                 rmv_port_callback, (void *)(intptr_t)port_id))
3555                         fprintf(stderr,
3556                                 "Could not set up deferred device removal\n");
3557                 break;
3558         case RTE_ETH_EVENT_DESTROY:
3559                 ports[port_id].port_status = RTE_PORT_CLOSED;
3560                 printf("Port %u is closed\n", port_id);
3561                 break;
3562         default:
3563                 break;
3564         }
3565         return 0;
3566 }
3567
3568 static int
3569 register_eth_event_callback(void)
3570 {
3571         int ret;
3572         enum rte_eth_event_type event;
3573
3574         for (event = RTE_ETH_EVENT_UNKNOWN;
3575                         event < RTE_ETH_EVENT_MAX; event++) {
3576                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3577                                 event,
3578                                 eth_event_callback,
3579                                 NULL);
3580                 if (ret != 0) {
3581                         TESTPMD_LOG(ERR, "Failed to register callback for "
3582                                         "%s event\n", eth_event_desc[event]);
3583                         return -1;
3584                 }
3585         }
3586
3587         return 0;
3588 }
3589
3590 /* This function is used by the interrupt thread */
3591 static void
3592 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3593                              __rte_unused void *arg)
3594 {
3595         uint16_t port_id;
3596         int ret;
3597
3598         if (type >= RTE_DEV_EVENT_MAX) {
3599                 fprintf(stderr, "%s called upon invalid event %d\n",
3600                         __func__, type);
3601                 fflush(stderr);
3602         }
3603
3604         switch (type) {
3605         case RTE_DEV_EVENT_REMOVE:
3606                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3607                         device_name);
3608                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3609                 if (ret) {
3610                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3611                                 device_name);
3612                         return;
3613                 }
3614                 /*
3615                  * Because the user's callback is invoked in eal interrupt
3616                  * callback, the interrupt callback need to be finished before
3617                  * it can be unregistered when detaching device. So finish
3618                  * callback soon and use a deferred removal to detach device
3619                  * is need. It is a workaround, once the device detaching be
3620                  * moved into the eal in the future, the deferred removal could
3621                  * be deleted.
3622                  */
3623                 if (rte_eal_alarm_set(100000,
3624                                 rmv_port_callback, (void *)(intptr_t)port_id))
3625                         RTE_LOG(ERR, EAL,
3626                                 "Could not set up deferred device removal\n");
3627                 break;
3628         case RTE_DEV_EVENT_ADD:
3629                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3630                         device_name);
3631                 /* TODO: After finish kernel driver binding,
3632                  * begin to attach port.
3633                  */
3634                 break;
3635         default:
3636                 break;
3637         }
3638 }
3639
3640 static void
3641 rxtx_port_config(portid_t pid)
3642 {
3643         uint16_t qid;
3644         uint64_t offloads;
3645         struct rte_port *port = &ports[pid];
3646
3647         for (qid = 0; qid < nb_rxq; qid++) {
3648                 offloads = port->rx_conf[qid].offloads;
3649                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3650
3651                 if (rxq_share > 0 &&
3652                     (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3653                         /* Non-zero share group to enable RxQ share. */
3654                         port->rx_conf[qid].share_group = pid / rxq_share + 1;
3655                         port->rx_conf[qid].share_qid = qid; /* Equal mapping. */
3656                 }
3657
3658                 if (offloads != 0)
3659                         port->rx_conf[qid].offloads = offloads;
3660
3661                 /* Check if any Rx parameters have been passed */
3662                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3663                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3664
3665                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3666                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3667
3668                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3669                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3670
3671                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3672                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3673
3674                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3675                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3676
3677                 port->nb_rx_desc[qid] = nb_rxd;
3678         }
3679
3680         for (qid = 0; qid < nb_txq; qid++) {
3681                 offloads = port->tx_conf[qid].offloads;
3682                 port->tx_conf[qid] = port->dev_info.default_txconf;
3683                 if (offloads != 0)
3684                         port->tx_conf[qid].offloads = offloads;
3685
3686                 /* Check if any Tx parameters have been passed */
3687                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3688                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3689
3690                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3691                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3692
3693                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3694                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3695
3696                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3697                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3698
3699                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3700                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3701
3702                 port->nb_tx_desc[qid] = nb_txd;
3703         }
3704 }
3705
3706 /*
3707  * Helper function to set MTU from frame size
3708  *
3709  * port->dev_info should be set before calling this function.
3710  *
3711  * return 0 on success, negative on error
3712  */
3713 int
3714 update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3715 {
3716         struct rte_port *port = &ports[portid];
3717         uint32_t eth_overhead;
3718         uint16_t mtu, new_mtu;
3719
3720         eth_overhead = get_eth_overhead(&port->dev_info);
3721
3722         if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3723                 printf("Failed to get MTU for port %u\n", portid);
3724                 return -1;
3725         }
3726
3727         new_mtu = max_rx_pktlen - eth_overhead;
3728
3729         if (mtu == new_mtu)
3730                 return 0;
3731
3732         if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3733                 fprintf(stderr,
3734                         "Failed to set MTU to %u for port %u\n",
3735                         new_mtu, portid);
3736                 return -1;
3737         }
3738
3739         port->dev_conf.rxmode.mtu = new_mtu;
3740
3741         return 0;
3742 }
3743
3744 void
3745 init_port_config(void)
3746 {
3747         portid_t pid;
3748         struct rte_port *port;
3749         int ret, i;
3750
3751         RTE_ETH_FOREACH_DEV(pid) {
3752                 port = &ports[pid];
3753                 port->dev_conf.fdir_conf = fdir_conf;
3754
3755                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3756                 if (ret != 0)
3757                         return;
3758
3759                 if (nb_rxq > 1) {
3760                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3761                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3762                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3763                 } else {
3764                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3765                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3766                 }
3767
3768                 if (port->dcb_flag == 0) {
3769                         if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3770                                 port->dev_conf.rxmode.mq_mode =
3771                                         (enum rte_eth_rx_mq_mode)
3772                                                 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3773                         } else {
3774                                 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3775                                 port->dev_conf.rxmode.offloads &=
3776                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3777
3778                                 for (i = 0;
3779                                      i < port->dev_info.nb_rx_queues;
3780                                      i++)
3781                                         port->rx_conf[i].offloads &=
3782                                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3783                         }
3784                 }
3785
3786                 rxtx_port_config(pid);
3787
3788                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3789                 if (ret != 0)
3790                         return;
3791
3792 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3793                 rte_pmd_ixgbe_bypass_init(pid);
3794 #endif
3795
3796                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3797                         port->dev_conf.intr_conf.lsc = 1;
3798                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3799                         port->dev_conf.intr_conf.rmv = 1;
3800         }
3801 }
3802
3803 void set_port_slave_flag(portid_t slave_pid)
3804 {
3805         struct rte_port *port;
3806
3807         port = &ports[slave_pid];
3808         port->slave_flag = 1;
3809 }
3810
3811 void clear_port_slave_flag(portid_t slave_pid)
3812 {
3813         struct rte_port *port;
3814
3815         port = &ports[slave_pid];
3816         port->slave_flag = 0;
3817 }
3818
3819 uint8_t port_is_bonding_slave(portid_t slave_pid)
3820 {
3821         struct rte_port *port;
3822         struct rte_eth_dev_info dev_info;
3823         int ret;
3824
3825         port = &ports[slave_pid];
3826         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3827         if (ret != 0) {
3828                 TESTPMD_LOG(ERR,
3829                         "Failed to get device info for port id %d,"
3830                         "cannot determine if the port is a bonded slave",
3831                         slave_pid);
3832                 return 0;
3833         }
3834         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3835                 return 1;
3836         return 0;
3837 }
3838
3839 const uint16_t vlan_tags[] = {
3840                 0,  1,  2,  3,  4,  5,  6,  7,
3841                 8,  9, 10, 11,  12, 13, 14, 15,
3842                 16, 17, 18, 19, 20, 21, 22, 23,
3843                 24, 25, 26, 27, 28, 29, 30, 31
3844 };
3845
3846 static  int
3847 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3848                  enum dcb_mode_enable dcb_mode,
3849                  enum rte_eth_nb_tcs num_tcs,
3850                  uint8_t pfc_en)
3851 {
3852         uint8_t i;
3853         int32_t rc;
3854         struct rte_eth_rss_conf rss_conf;
3855
3856         /*
3857          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3858          * given above, and the number of traffic classes available for use.
3859          */
3860         if (dcb_mode == DCB_VT_ENABLED) {
3861                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3862                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3863                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3864                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3865
3866                 /* VMDQ+DCB RX and TX configurations */
3867                 vmdq_rx_conf->enable_default_pool = 0;
3868                 vmdq_rx_conf->default_pool = 0;
3869                 vmdq_rx_conf->nb_queue_pools =
3870                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3871                 vmdq_tx_conf->nb_queue_pools =
3872                         (num_tcs ==  RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3873
3874                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3875                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3876                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3877                         vmdq_rx_conf->pool_map[i].pools =
3878                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3879                 }
3880                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3881                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3882                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3883                 }
3884
3885                 /* set DCB mode of RX and TX of multiple queues */
3886                 eth_conf->rxmode.mq_mode =
3887                                 (enum rte_eth_rx_mq_mode)
3888                                         (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
3889                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
3890         } else {
3891                 struct rte_eth_dcb_rx_conf *rx_conf =
3892                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3893                 struct rte_eth_dcb_tx_conf *tx_conf =
3894                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3895
3896                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3897
3898                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3899                 if (rc != 0)
3900                         return rc;
3901
3902                 rx_conf->nb_tcs = num_tcs;
3903                 tx_conf->nb_tcs = num_tcs;
3904
3905                 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3906                         rx_conf->dcb_tc[i] = i % num_tcs;
3907                         tx_conf->dcb_tc[i] = i % num_tcs;
3908                 }
3909
3910                 eth_conf->rxmode.mq_mode =
3911                                 (enum rte_eth_rx_mq_mode)
3912                                         (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
3913                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3914                 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
3915         }
3916
3917         if (pfc_en)
3918                 eth_conf->dcb_capability_en =
3919                                 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
3920         else
3921                 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
3922
3923         return 0;
3924 }
3925
3926 int
3927 init_port_dcb_config(portid_t pid,
3928                      enum dcb_mode_enable dcb_mode,
3929                      enum rte_eth_nb_tcs num_tcs,
3930                      uint8_t pfc_en)
3931 {
3932         struct rte_eth_conf port_conf;
3933         struct rte_port *rte_port;
3934         int retval;
3935         uint16_t i;
3936
3937         if (num_procs > 1) {
3938                 printf("The multi-process feature doesn't support dcb.\n");
3939                 return -ENOTSUP;
3940         }
3941         rte_port = &ports[pid];
3942
3943         /* retain the original device configuration. */
3944         memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
3945
3946         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3947         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3948         if (retval < 0)
3949                 return retval;
3950         port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
3951         /* remove RSS HASH offload for DCB in vt mode */
3952         if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
3953                 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3954                 for (i = 0; i < nb_rxq; i++)
3955                         rte_port->rx_conf[i].offloads &=
3956                                 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3957         }
3958
3959         /* re-configure the device . */
3960         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3961         if (retval < 0)
3962                 return retval;
3963
3964         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3965         if (retval != 0)
3966                 return retval;
3967
3968         /* If dev_info.vmdq_pool_base is greater than 0,
3969          * the queue id of vmdq pools is started after pf queues.
3970          */
3971         if (dcb_mode == DCB_VT_ENABLED &&
3972             rte_port->dev_info.vmdq_pool_base > 0) {
3973                 fprintf(stderr,
3974                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
3975                         pid);
3976                 return -1;
3977         }
3978
3979         /* Assume the ports in testpmd have the same dcb capability
3980          * and has the same number of rxq and txq in dcb mode
3981          */
3982         if (dcb_mode == DCB_VT_ENABLED) {
3983                 if (rte_port->dev_info.max_vfs > 0) {
3984                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3985                         nb_txq = rte_port->dev_info.nb_tx_queues;
3986                 } else {
3987                         nb_rxq = rte_port->dev_info.max_rx_queues;
3988                         nb_txq = rte_port->dev_info.max_tx_queues;
3989                 }
3990         } else {
3991                 /*if vt is disabled, use all pf queues */
3992                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3993                         nb_rxq = rte_port->dev_info.max_rx_queues;
3994                         nb_txq = rte_port->dev_info.max_tx_queues;
3995                 } else {
3996                         nb_rxq = (queueid_t)num_tcs;
3997                         nb_txq = (queueid_t)num_tcs;
3998
3999                 }
4000         }
4001         rx_free_thresh = 64;
4002
4003         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4004
4005         rxtx_port_config(pid);
4006         /* VLAN filter */
4007         rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4008         for (i = 0; i < RTE_DIM(vlan_tags); i++)
4009                 rx_vft_set(pid, vlan_tags[i], 1);
4010
4011         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4012         if (retval != 0)
4013                 return retval;
4014
4015         rte_port->dcb_flag = 1;
4016
4017         /* Enter DCB configuration status */
4018         dcb_config = 1;
4019
4020         return 0;
4021 }
4022
4023 static void
4024 init_port(void)
4025 {
4026         int i;
4027
4028         /* Configuration of Ethernet ports. */
4029         ports = rte_zmalloc("testpmd: ports",
4030                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4031                             RTE_CACHE_LINE_SIZE);
4032         if (ports == NULL) {
4033                 rte_exit(EXIT_FAILURE,
4034                                 "rte_zmalloc(%d struct rte_port) failed\n",
4035                                 RTE_MAX_ETHPORTS);
4036         }
4037         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4038                 ports[i].xstats_info.allocated = false;
4039         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4040                 LIST_INIT(&ports[i].flow_tunnel_list);
4041         /* Initialize ports NUMA structures */
4042         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4043         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4044         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4045 }
4046
4047 static void
4048 force_quit(void)
4049 {
4050         pmd_test_exit();
4051         prompt_exit();
4052 }
4053
4054 static void
4055 print_stats(void)
4056 {
4057         uint8_t i;
4058         const char clr[] = { 27, '[', '2', 'J', '\0' };
4059         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4060
4061         /* Clear screen and move to top left */
4062         printf("%s%s", clr, top_left);
4063
4064         printf("\nPort statistics ====================================");
4065         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4066                 nic_stats_display(fwd_ports_ids[i]);
4067
4068         fflush(stdout);
4069 }
4070
4071 static void
4072 signal_handler(int signum)
4073 {
4074         if (signum == SIGINT || signum == SIGTERM) {
4075                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4076                         signum);
4077 #ifdef RTE_LIB_PDUMP
4078                 /* uninitialize packet capture framework */
4079                 rte_pdump_uninit();
4080 #endif
4081 #ifdef RTE_LIB_LATENCYSTATS
4082                 if (latencystats_enabled != 0)
4083                         rte_latencystats_uninit();
4084 #endif
4085                 force_quit();
4086                 /* Set flag to indicate the force termination. */
4087                 f_quit = 1;
4088                 /* exit with the expected status */
4089 #ifndef RTE_EXEC_ENV_WINDOWS
4090                 signal(signum, SIG_DFL);
4091                 kill(getpid(), signum);
4092 #endif
4093         }
4094 }
4095
4096 int
4097 main(int argc, char** argv)
4098 {
4099         int diag;
4100         portid_t port_id;
4101         uint16_t count;
4102         int ret;
4103
4104         signal(SIGINT, signal_handler);
4105         signal(SIGTERM, signal_handler);
4106
4107         testpmd_logtype = rte_log_register("testpmd");
4108         if (testpmd_logtype < 0)
4109                 rte_exit(EXIT_FAILURE, "Cannot register log type");
4110         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4111
4112         diag = rte_eal_init(argc, argv);
4113         if (diag < 0)
4114                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4115                          rte_strerror(rte_errno));
4116
4117         ret = register_eth_event_callback();
4118         if (ret != 0)
4119                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4120
4121 #ifdef RTE_LIB_PDUMP
4122         /* initialize packet capture framework */
4123         rte_pdump_init();
4124 #endif
4125
4126         count = 0;
4127         RTE_ETH_FOREACH_DEV(port_id) {
4128                 ports_ids[count] = port_id;
4129                 count++;
4130         }
4131         nb_ports = (portid_t) count;
4132         if (nb_ports == 0)
4133                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4134
4135         /* allocate port structures, and init them */
4136         init_port();
4137
4138         set_def_fwd_config();
4139         if (nb_lcores == 0)
4140                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4141                          "Check the core mask argument\n");
4142
4143         /* Bitrate/latency stats disabled by default */
4144 #ifdef RTE_LIB_BITRATESTATS
4145         bitrate_enabled = 0;
4146 #endif
4147 #ifdef RTE_LIB_LATENCYSTATS
4148         latencystats_enabled = 0;
4149 #endif
4150
4151         /* on FreeBSD, mlockall() is disabled by default */
4152 #ifdef RTE_EXEC_ENV_FREEBSD
4153         do_mlockall = 0;
4154 #else
4155         do_mlockall = 1;
4156 #endif
4157
4158         argc -= diag;
4159         argv += diag;
4160         if (argc > 1)
4161                 launch_args_parse(argc, argv);
4162
4163 #ifndef RTE_EXEC_ENV_WINDOWS
4164         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4165                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4166                         strerror(errno));
4167         }
4168 #endif
4169
4170         if (tx_first && interactive)
4171                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4172                                 "interactive mode.\n");
4173
4174         if (tx_first && lsc_interrupt) {
4175                 fprintf(stderr,
4176                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4177                 lsc_interrupt = 0;
4178         }
4179
4180         if (!nb_rxq && !nb_txq)
4181                 fprintf(stderr,
4182                         "Warning: Either rx or tx queues should be non-zero\n");
4183
4184         if (nb_rxq > 1 && nb_rxq > nb_txq)
4185                 fprintf(stderr,
4186                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4187                         nb_rxq, nb_txq);
4188
4189         init_config();
4190
4191         if (hot_plug) {
4192                 ret = rte_dev_hotplug_handle_enable();
4193                 if (ret) {
4194                         RTE_LOG(ERR, EAL,
4195                                 "fail to enable hotplug handling.");
4196                         return -1;
4197                 }
4198
4199                 ret = rte_dev_event_monitor_start();
4200                 if (ret) {
4201                         RTE_LOG(ERR, EAL,
4202                                 "fail to start device event monitoring.");
4203                         return -1;
4204                 }
4205
4206                 ret = rte_dev_event_callback_register(NULL,
4207                         dev_event_callback, NULL);
4208                 if (ret) {
4209                         RTE_LOG(ERR, EAL,
4210                                 "fail  to register device event callback\n");
4211                         return -1;
4212                 }
4213         }
4214
4215         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4216                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4217
4218         /* set all ports to promiscuous mode by default */
4219         RTE_ETH_FOREACH_DEV(port_id) {
4220                 ret = rte_eth_promiscuous_enable(port_id);
4221                 if (ret != 0)
4222                         fprintf(stderr,
4223                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4224                                 port_id, rte_strerror(-ret));
4225         }
4226
4227 #ifdef RTE_LIB_METRICS
4228         /* Init metrics library */
4229         rte_metrics_init(rte_socket_id());
4230 #endif
4231
4232 #ifdef RTE_LIB_LATENCYSTATS
4233         if (latencystats_enabled != 0) {
4234                 int ret = rte_latencystats_init(1, NULL);
4235                 if (ret)
4236                         fprintf(stderr,
4237                                 "Warning: latencystats init() returned error %d\n",
4238                                 ret);
4239                 fprintf(stderr, "Latencystats running on lcore %d\n",
4240                         latencystats_lcore_id);
4241         }
4242 #endif
4243
4244         /* Setup bitrate stats */
4245 #ifdef RTE_LIB_BITRATESTATS
4246         if (bitrate_enabled != 0) {
4247                 bitrate_data = rte_stats_bitrate_create();
4248                 if (bitrate_data == NULL)
4249                         rte_exit(EXIT_FAILURE,
4250                                 "Could not allocate bitrate data.\n");
4251                 rte_stats_bitrate_reg(bitrate_data);
4252         }
4253 #endif
4254 #ifdef RTE_LIB_CMDLINE
4255         if (strlen(cmdline_filename) != 0)
4256                 cmdline_read_from_file(cmdline_filename);
4257
4258         if (interactive == 1) {
4259                 if (auto_start) {
4260                         printf("Start automatic packet forwarding\n");
4261                         start_packet_forwarding(0);
4262                 }
4263                 prompt();
4264                 pmd_test_exit();
4265         } else
4266 #endif
4267         {
4268                 char c;
4269                 int rc;
4270
4271                 f_quit = 0;
4272
4273                 printf("No commandline core given, start packet forwarding\n");
4274                 start_packet_forwarding(tx_first);
4275                 if (stats_period != 0) {
4276                         uint64_t prev_time = 0, cur_time, diff_time = 0;
4277                         uint64_t timer_period;
4278
4279                         /* Convert to number of cycles */
4280                         timer_period = stats_period * rte_get_timer_hz();
4281
4282                         while (f_quit == 0) {
4283                                 cur_time = rte_get_timer_cycles();
4284                                 diff_time += cur_time - prev_time;
4285
4286                                 if (diff_time >= timer_period) {
4287                                         print_stats();
4288                                         /* Reset the timer */
4289                                         diff_time = 0;
4290                                 }
4291                                 /* Sleep to avoid unnecessary checks */
4292                                 prev_time = cur_time;
4293                                 rte_delay_us_sleep(US_PER_S);
4294                         }
4295                 }
4296
4297                 printf("Press enter to exit\n");
4298                 rc = read(0, &c, 1);
4299                 pmd_test_exit();
4300                 if (rc < 0)
4301                         return 1;
4302         }
4303
4304         ret = rte_eal_cleanup();
4305         if (ret != 0)
4306                 rte_exit(EXIT_FAILURE,
4307                          "EAL cleanup failed: %s\n", strerror(-ret));
4308
4309         return EXIT_SUCCESS;
4310 }