app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #ifndef RTE_EXEC_ENV_WINDOWS
  13 #include <sys/mman.h>
  14 #endif
  15 #include <sys/types.h>
  16 #include <errno.h>
  17 #include <stdbool.h>
  18
  19 #include <sys/queue.h>
  20 #include <sys/stat.h>
  21
  22 #include <stdint.h>
  23 #include <unistd.h>
  24 #include <inttypes.h>
  25
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_byteorder.h>
  29 #include <rte_log.h>
  30 #include <rte_debug.h>
  31 #include <rte_cycles.h>
  32 #include <rte_memory.h>
  33 #include <rte_memcpy.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_alarm.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_atomic.h>
  40 #include <rte_branch_prediction.h>
  41 #include <rte_mempool.h>
  42 #include <rte_malloc.h>
  43 #include <rte_mbuf.h>
  44 #include <rte_mbuf_pool_ops.h>
  45 #include <rte_interrupts.h>
  46 #include <rte_pci.h>
  47 #include <rte_ether.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_dev.h>
  50 #include <rte_string_fns.h>
  51 #ifdef RTE_NET_IXGBE
  52 #include <rte_pmd_ixgbe.h>
  53 #endif
  54 #ifdef RTE_LIB_PDUMP
  55 #include <rte_pdump.h>
  56 #endif
  57 #include <rte_flow.h>
  58 #include <rte_metrics.h>
  59 #ifdef RTE_LIB_BITRATESTATS
  60 #include <rte_bitrate.h>
  61 #endif
  62 #ifdef RTE_LIB_LATENCYSTATS
  63 #include <rte_latencystats.h>
  64 #endif
  65 #ifdef RTE_EXEC_ENV_WINDOWS
  66 #include <process.h>
  67 #endif
  68
  69 #include "testpmd.h"
  70
  71 #ifndef MAP_HUGETLB
  72 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  73 #define HUGE_FLAG (0x40000)
  74 #else
  75 #define HUGE_FLAG MAP_HUGETLB
  76 #endif
  77
  78 #ifndef MAP_HUGE_SHIFT
  79 /* older kernels (or FreeBSD) will not have this define */
  80 #define HUGE_SHIFT (26)
  81 #else
  82 #define HUGE_SHIFT MAP_HUGE_SHIFT
  83 #endif
  84
  85 #define EXTMEM_HEAP_NAME "extmem"
  86 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  87
  88 uint16_t verbose_level = 0; /**< Silent by default. */
  89 int testpmd_logtype; /**< Log type for testpmd logs */
  90
  91 /* use main core for command line ? */
  92 uint8_t interactive = 0;
  93 uint8_t auto_start = 0;
  94 uint8_t tx_first;
  95 char cmdline_filename[PATH_MAX] = {0};
  96
  97 /*
  98  * NUMA support configuration.
  99  * When set, the NUMA support attempts to dispatch the allocation of the
 100  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
 101  * probed ports among the CPU sockets 0 and 1.
 102  * Otherwise, all memory is allocated from CPU socket 0.
 103  */
 104 uint8_t numa_support = 1; /**< numa enabled by default */
 105
 106 /*
 107  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 108  * not configured.
 109  */
 110 uint8_t socket_num = UMA_NO_CONFIG;
 111
 112 /*
 113  * Select mempool allocation type:
 114  * - native: use regular DPDK memory
 115  * - anon: use regular DPDK memory to create mempool, but populate using
 116  *         anonymous memory (may not be IOVA-contiguous)
 117  * - xmem: use externally allocated hugepage memory
 118  */
 119 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 120
 121 /*
 122  * Store specified sockets on which memory pool to be used by ports
 123  * is allocated.
 124  */
 125 uint8_t port_numa[RTE_MAX_ETHPORTS];
 126
 127 /*
 128  * Store specified sockets on which RX ring to be used by ports
 129  * is allocated.
 130  */
 131 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 132
 133 /*
 134  * Store specified sockets on which TX ring to be used by ports
 135  * is allocated.
 136  */
 137 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 138
 139 /*
 140  * Record the Ethernet address of peer target ports to which packets are
 141  * forwarded.
 142  * Must be instantiated with the ethernet addresses of peer traffic generator
 143  * ports.
 144  */
 145 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 146 portid_t nb_peer_eth_addrs = 0;
 147
 148 /*
 149  * Probed Target Environment.
 150  */
 151 struct rte_port *ports;        /**< For all probed ethernet ports. */
 152 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 153 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 154 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 155
 156 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 157
 158 /*
 159  * Test Forwarding Configuration.
 160  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 161  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 162  */
 163 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 164 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 165 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 166 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 167
 168 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 169 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 170
 171 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 172 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 173
 174 /*
 175  * Forwarding engines.
 176  */
 177 struct fwd_engine * fwd_engines[] = {
 178         &io_fwd_engine,
 179         &mac_fwd_engine,
 180         &mac_swap_engine,
 181         &flow_gen_engine,
 182         &rx_only_engine,
 183         &tx_only_engine,
 184         &csum_fwd_engine,
 185         &icmp_echo_engine,
 186         &noisy_vnf_engine,
 187         &five_tuple_swap_fwd_engine,
 188 #ifdef RTE_LIBRTE_IEEE1588
 189         &ieee1588_fwd_engine,
 190 #endif
 191         NULL,
 192 };
 193
 194 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 195 uint16_t mempool_flags;
 196
 197 struct fwd_config cur_fwd_config;
 198 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 199 uint32_t retry_enabled;
 200 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 201 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 202
 203 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 204 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 205         DEFAULT_MBUF_DATA_SIZE
 206 }; /**< Mbuf data space size. */
 207 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 208                                       * specified on command-line. */
 209 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 210
 211 /*
 212  * In container, it cannot terminate the process which running with 'stats-period'
 213  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 214  */
 215 uint8_t f_quit;
 216
 217 /*
 218  * Configuration of packet segments used to scatter received packets
 219  * if some of split features is configured.
 220  */
 221 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 222 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 223 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 224 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 225
 226 /*
 227  * Configuration of packet segments used by the "txonly" processing engine.
 228  */
 229 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 230 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 231         TXONLY_DEF_PACKET_LEN,
 232 };
 233 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 234
 235 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 236 /**< Split policy for packets to TX. */
 237
 238 uint8_t txonly_multi_flow;
 239 /**< Whether multiple flows are generated in TXONLY mode. */
 240
 241 uint32_t tx_pkt_times_inter;
 242 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 243
 244 uint32_t tx_pkt_times_intra;
 245 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 246
 247 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 248 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 249 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 250
 251 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 252 uint8_t dcb_config = 0;
 253
 254 /*
 255  * Configurable number of RX/TX queues.
 256  */
 257 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 258 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 259 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 260
 261 /*
 262  * Configurable number of RX/TX ring descriptors.
 263  * Defaults are supplied by drivers via ethdev.
 264  */
 265 #define RTE_TEST_RX_DESC_DEFAULT 0
 266 #define RTE_TEST_TX_DESC_DEFAULT 0
 267 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 268 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 269
 270 #define RTE_PMD_PARAM_UNSET -1
 271 /*
 272  * Configurable values of RX and TX ring threshold registers.
 273  */
 274
 275 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 276 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 277 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 278
 279 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 280 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 281 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of RX free threshold.
 285  */
 286 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of RX drop enable.
 290  */
 291 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 292
 293 /*
 294  * Configurable value of TX free threshold.
 295  */
 296 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 297
 298 /*
 299  * Configurable value of TX RS bit threshold.
 300  */
 301 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 302
 303 /*
 304  * Configurable value of buffered packets before sending.
 305  */
 306 uint16_t noisy_tx_sw_bufsz;
 307
 308 /*
 309  * Configurable value of packet buffer timeout.
 310  */
 311 uint16_t noisy_tx_sw_buf_flush_time;
 312
 313 /*
 314  * Configurable value for size of VNF internal memory area
 315  * used for simulating noisy neighbour behaviour
 316  */
 317 uint64_t noisy_lkup_mem_sz;
 318
 319 /*
 320  * Configurable value of number of random writes done in
 321  * VNF simulation memory area.
 322  */
 323 uint64_t noisy_lkup_num_writes;
 324
 325 /*
 326  * Configurable value of number of random reads done in
 327  * VNF simulation memory area.
 328  */
 329 uint64_t noisy_lkup_num_reads;
 330
 331 /*
 332  * Configurable value of number of random reads/writes done in
 333  * VNF simulation memory area.
 334  */
 335 uint64_t noisy_lkup_num_reads_writes;
 336
 337 /*
 338  * Receive Side Scaling (RSS) configuration.
 339  */
 340 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 341
 342 /*
 343  * Port topology configuration
 344  */
 345 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 346
 347 /*
 348  * Avoids to flush all the RX streams before starts forwarding.
 349  */
 350 uint8_t no_flush_rx = 0; /* flush by default */
 351
 352 /*
 353  * Flow API isolated mode.
 354  */
 355 uint8_t flow_isolate_all;
 356
 357 /*
 358  * Avoids to check link status when starting/stopping a port.
 359  */
 360 uint8_t no_link_check = 0; /* check by default */
 361
 362 /*
 363  * Don't automatically start all ports in interactive mode.
 364  */
 365 uint8_t no_device_start = 0;
 366
 367 /*
 368  * Enable link status change notification
 369  */
 370 uint8_t lsc_interrupt = 1; /* enabled by default */
 371
 372 /*
 373  * Enable device removal notification.
 374  */
 375 uint8_t rmv_interrupt = 1; /* enabled by default */
 376
 377 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 378
 379 /* After attach, port setup is called on event or by iterator */
 380 bool setup_on_probe_event = true;
 381
 382 /* Clear ptypes on port initialization. */
 383 uint8_t clear_ptypes = true;
 384
 385 /* Hairpin ports configuration mode. */
 386 uint16_t hairpin_mode;
 387
 388 /* Pretty printing of ethdev events */
 389 static const char * const eth_event_desc[] = {
 390         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 391         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 392         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 393         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 394         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 395         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 396         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 397         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 398         [RTE_ETH_EVENT_NEW] = "device probed",
 399         [RTE_ETH_EVENT_DESTROY] = "device released",
 400         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 401         [RTE_ETH_EVENT_MAX] = NULL,
 402 };
 403
 404 /*
 405  * Display or mask ether events
 406  * Default to all events except VF_MBOX
 407  */
 408 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 414                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 415                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 416 /*
 417  * Decide if all memory are locked for performance.
 418  */
 419 int do_mlockall = 0;
 420
 421 /*
 422  * NIC bypass mode configuration options.
 423  */
 424
 425 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 426 /* The NIC bypass watchdog timeout. */
 427 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 428 #endif
 429
 430
 431 #ifdef RTE_LIB_LATENCYSTATS
 432
 433 /*
 434  * Set when latency stats is enabled in the commandline
 435  */
 436 uint8_t latencystats_enabled;
 437
 438 /*
 439  * Lcore ID to serive latency statistics.
 440  */
 441 lcoreid_t latencystats_lcore_id = -1;
 442
 443 #endif
 444
 445 /*
 446  * Ethernet device configuration.
 447  */
 448 struct rte_eth_rxmode rx_mode = {
 449         /* Default maximum frame length.
 450          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 451          * in init_config().
 452          */
 453         .max_rx_pkt_len = 0,
 454 };
 455
 456 struct rte_eth_txmode tx_mode = {
 457         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 458 };
 459
 460 struct rte_fdir_conf fdir_conf = {
 461         .mode = RTE_FDIR_MODE_NONE,
 462         .pballoc = RTE_FDIR_PBALLOC_64K,
 463         .status = RTE_FDIR_REPORT_STATUS,
 464         .mask = {
 465                 .vlan_tci_mask = 0xFFEF,
 466                 .ipv4_mask     = {
 467                         .src_ip = 0xFFFFFFFF,
 468                         .dst_ip = 0xFFFFFFFF,
 469                 },
 470                 .ipv6_mask     = {
 471                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 472                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 473                 },
 474                 .src_port_mask = 0xFFFF,
 475                 .dst_port_mask = 0xFFFF,
 476                 .mac_addr_byte_mask = 0xFF,
 477                 .tunnel_type_mask = 1,
 478                 .tunnel_id_mask = 0xFFFFFFFF,
 479         },
 480         .drop_queue = 127,
 481 };
 482
 483 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 484
 485 /*
 486  * Display zero values by default for xstats
 487  */
 488 uint8_t xstats_hide_zero;
 489
 490 /*
 491  * Measure of CPU cycles disabled by default
 492  */
 493 uint8_t record_core_cycles;
 494
 495 /*
 496  * Display of RX and TX bursts disabled by default
 497  */
 498 uint8_t record_burst_stats;
 499
 500 unsigned int num_sockets = 0;
 501 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 502
 503 #ifdef RTE_LIB_BITRATESTATS
 504 /* Bitrate statistics */
 505 struct rte_stats_bitrates *bitrate_data;
 506 lcoreid_t bitrate_lcore_id;
 507 uint8_t bitrate_enabled;
 508 #endif
 509
 510 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 511 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 512
 513 /*
 514  * hexadecimal bitmask of RX mq mode can be enabled.
 515  */
 516 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 517
 518 /*
 519  * Used to set forced link speed
 520  */
 521 uint32_t eth_link_speed;
 522
 523 /* Forward function declarations */
 524 static void setup_attached_port(portid_t pi);
 525 static void check_all_ports_link_status(uint32_t port_mask);
 526 static int eth_event_callback(portid_t port_id,
 527                               enum rte_eth_event_type type,
 528                               void *param, void *ret_param);
 529 static void dev_event_callback(const char *device_name,
 530                                 enum rte_dev_event_type type,
 531                                 void *param);
 532
 533 /*
 534  * Check if all the ports are started.
 535  * If yes, return positive value. If not, return zero.
 536  */
 537 static int all_ports_started(void);
 538
 539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 541
 542 /* Holds the registered mbuf dynamic flags names. */
 543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 544
 545 /*
 546  * Helper function to check if socket is already discovered.
 547  * If yes, return positive value. If not, return zero.
 548  */
 549 int
 550 new_socket_id(unsigned int socket_id)
 551 {
 552         unsigned int i;
 553
 554         for (i = 0; i < num_sockets; i++) {
 555                 if (socket_ids[i] == socket_id)
 556                         return 0;
 557         }
 558         return 1;
 559 }
 560
 561 /*
 562  * Setup default configuration.
 563  */
 564 static void
 565 set_default_fwd_lcores_config(void)
 566 {
 567         unsigned int i;
 568         unsigned int nb_lc;
 569         unsigned int sock_num;
 570
 571         nb_lc = 0;
 572         for (i = 0; i < RTE_MAX_LCORE; i++) {
 573                 if (!rte_lcore_is_enabled(i))
 574                         continue;
 575                 sock_num = rte_lcore_to_socket_id(i);
 576                 if (new_socket_id(sock_num)) {
 577                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 578                                 rte_exit(EXIT_FAILURE,
 579                                          "Total sockets greater than %u\n",
 580                                          RTE_MAX_NUMA_NODES);
 581                         }
 582                         socket_ids[num_sockets++] = sock_num;
 583                 }
 584                 if (i == rte_get_main_lcore())
 585                         continue;
 586                 fwd_lcores_cpuids[nb_lc++] = i;
 587         }
 588         nb_lcores = (lcoreid_t) nb_lc;
 589         nb_cfg_lcores = nb_lcores;
 590         nb_fwd_lcores = 1;
 591 }
 592
 593 static void
 594 set_def_peer_eth_addrs(void)
 595 {
 596         portid_t i;
 597
 598         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 599                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 600                 peer_eth_addrs[i].addr_bytes[5] = i;
 601         }
 602 }
 603
 604 static void
 605 set_default_fwd_ports_config(void)
 606 {
 607         portid_t pt_id;
 608         int i = 0;
 609
 610         RTE_ETH_FOREACH_DEV(pt_id) {
 611                 fwd_ports_ids[i++] = pt_id;
 612
 613                 /* Update sockets info according to the attached device */
 614                 int socket_id = rte_eth_dev_socket_id(pt_id);
 615                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 616                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 617                                 rte_exit(EXIT_FAILURE,
 618                                          "Total sockets greater than %u\n",
 619                                          RTE_MAX_NUMA_NODES);
 620                         }
 621                         socket_ids[num_sockets++] = socket_id;
 622                 }
 623         }
 624
 625         nb_cfg_ports = nb_ports;
 626         nb_fwd_ports = nb_ports;
 627 }
 628
 629 void
 630 set_def_fwd_config(void)
 631 {
 632         set_default_fwd_lcores_config();
 633         set_def_peer_eth_addrs();
 634         set_default_fwd_ports_config();
 635 }
 636
 637 #ifndef RTE_EXEC_ENV_WINDOWS
 638 /* extremely pessimistic estimation of memory required to create a mempool */
 639 static int
 640 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 641 {
 642         unsigned int n_pages, mbuf_per_pg, leftover;
 643         uint64_t total_mem, mbuf_mem, obj_sz;
 644
 645         /* there is no good way to predict how much space the mempool will
 646          * occupy because it will allocate chunks on the fly, and some of those
 647          * will come from default DPDK memory while some will come from our
 648          * external memory, so just assume 128MB will be enough for everyone.
 649          */
 650         uint64_t hdr_mem = 128 << 20;
 651
 652         /* account for possible non-contiguousness */
 653         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 654         if (obj_sz > pgsz) {
 655                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 656                 return -1;
 657         }
 658
 659         mbuf_per_pg = pgsz / obj_sz;
 660         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 661         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 662
 663         mbuf_mem = n_pages * pgsz;
 664
 665         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 666
 667         if (total_mem > SIZE_MAX) {
 668                 TESTPMD_LOG(ERR, "Memory size too big\n");
 669                 return -1;
 670         }
 671         *out = (size_t)total_mem;
 672
 673         return 0;
 674 }
 675
 676 static int
 677 pagesz_flags(uint64_t page_sz)
 678 {
 679         /* as per mmap() manpage, all page sizes are log2 of page size
 680          * shifted by MAP_HUGE_SHIFT
 681          */
 682         int log2 = rte_log2_u64(page_sz);
 683
 684         return (log2 << HUGE_SHIFT);
 685 }
 686
 687 static void *
 688 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 689 {
 690         void *addr;
 691         int flags;
 692
 693         /* allocate anonymous hugepages */
 694         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 695         if (huge)
 696                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 697
 698         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 699         if (addr == MAP_FAILED)
 700                 return NULL;
 701
 702         return addr;
 703 }
 704
 705 struct extmem_param {
 706         void *addr;
 707         size_t len;
 708         size_t pgsz;
 709         rte_iova_t *iova_table;
 710         unsigned int iova_table_len;
 711 };
 712
 713 static int
 714 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 715                 bool huge)
 716 {
 717         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 718                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 719         unsigned int cur_page, n_pages, pgsz_idx;
 720         size_t mem_sz, cur_pgsz;
 721         rte_iova_t *iovas = NULL;
 722         void *addr;
 723         int ret;
 724
 725         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 726                 /* skip anything that is too big */
 727                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 728                         continue;
 729
 730                 cur_pgsz = pgsizes[pgsz_idx];
 731
 732                 /* if we were told not to allocate hugepages, override */
 733                 if (!huge)
 734                         cur_pgsz = sysconf(_SC_PAGESIZE);
 735
 736                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 737                 if (ret < 0) {
 738                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 739                         return -1;
 740                 }
 741
 742                 /* allocate our memory */
 743                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 744
 745                 /* if we couldn't allocate memory with a specified page size,
 746                  * that doesn't mean we can't do it with other page sizes, so
 747                  * try another one.
 748                  */
 749                 if (addr == NULL)
 750                         continue;
 751
 752                 /* store IOVA addresses for every page in this memory area */
 753                 n_pages = mem_sz / cur_pgsz;
 754
 755                 iovas = malloc(sizeof(*iovas) * n_pages);
 756
 757                 if (iovas == NULL) {
 758                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 759                         goto fail;
 760                 }
 761                 /* lock memory if it's not huge pages */
 762                 if (!huge)
 763                         mlock(addr, mem_sz);
 764
 765                 /* populate IOVA addresses */
 766                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 767                         rte_iova_t iova;
 768                         size_t offset;
 769                         void *cur;
 770
 771                         offset = cur_pgsz * cur_page;
 772                         cur = RTE_PTR_ADD(addr, offset);
 773
 774                         /* touch the page before getting its IOVA */
 775                         *(volatile char *)cur = 0;
 776
 777                         iova = rte_mem_virt2iova(cur);
 778
 779                         iovas[cur_page] = iova;
 780                 }
 781
 782                 break;
 783         }
 784         /* if we couldn't allocate anything */
 785         if (iovas == NULL)
 786                 return -1;
 787
 788         param->addr = addr;
 789         param->len = mem_sz;
 790         param->pgsz = cur_pgsz;
 791         param->iova_table = iovas;
 792         param->iova_table_len = n_pages;
 793
 794         return 0;
 795 fail:
 796         if (iovas)
 797                 free(iovas);
 798         if (addr)
 799                 munmap(addr, mem_sz);
 800
 801         return -1;
 802 }
 803
 804 static int
 805 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 806 {
 807         struct extmem_param param;
 808         int socket_id, ret;
 809
 810         memset(&param, 0, sizeof(param));
 811
 812         /* check if our heap exists */
 813         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 814         if (socket_id < 0) {
 815                 /* create our heap */
 816                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 817                 if (ret < 0) {
 818                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 819                         return -1;
 820                 }
 821         }
 822
 823         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 824         if (ret < 0) {
 825                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 826                 return -1;
 827         }
 828
 829         /* we now have a valid memory area, so add it to heap */
 830         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 831                         param.addr, param.len, param.iova_table,
 832                         param.iova_table_len, param.pgsz);
 833
 834         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 835
 836         /* not needed any more */
 837         free(param.iova_table);
 838
 839         if (ret < 0) {
 840                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 841                 munmap(param.addr, param.len);
 842                 return -1;
 843         }
 844
 845         /* success */
 846
 847         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 848                         param.len >> 20);
 849
 850         return 0;
 851 }
 852 static void
 853 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 854              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 855 {
 856         uint16_t pid = 0;
 857         int ret;
 858
 859         RTE_ETH_FOREACH_DEV(pid) {
 860                 struct rte_eth_dev_info dev_info;
 861
 862                 ret = eth_dev_info_get_print_err(pid, &dev_info);
 863                 if (ret != 0) {
 864                         TESTPMD_LOG(DEBUG,
 865                                     "unable to get device info for port %d on addr 0x%p,"
 866                                     "mempool unmapping will not be performed\n",
 867                                     pid, memhdr->addr);
 868                         continue;
 869                 }
 870
 871                 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
 872                 if (ret) {
 873                         TESTPMD_LOG(DEBUG,
 874                                     "unable to DMA unmap addr 0x%p "
 875                                     "for device %s\n",
 876                                     memhdr->addr, dev_info.device->name);
 877                 }
 878         }
 879         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 880         if (ret) {
 881                 TESTPMD_LOG(DEBUG,
 882                             "unable to un-register addr 0x%p\n", memhdr->addr);
 883         }
 884 }
 885
 886 static void
 887 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 888            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 889 {
 890         uint16_t pid = 0;
 891         size_t page_size = sysconf(_SC_PAGESIZE);
 892         int ret;
 893
 894         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 895                                   page_size);
 896         if (ret) {
 897                 TESTPMD_LOG(DEBUG,
 898                             "unable to register addr 0x%p\n", memhdr->addr);
 899                 return;
 900         }
 901         RTE_ETH_FOREACH_DEV(pid) {
 902                 struct rte_eth_dev_info dev_info;
 903
 904                 ret = eth_dev_info_get_print_err(pid, &dev_info);
 905                 if (ret != 0) {
 906                         TESTPMD_LOG(DEBUG,
 907                                     "unable to get device info for port %d on addr 0x%p,"
 908                                     "mempool mapping will not be performed\n",
 909                                     pid, memhdr->addr);
 910                         continue;
 911                 }
 912                 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
 913                 if (ret) {
 914                         TESTPMD_LOG(DEBUG,
 915                                     "unable to DMA map addr 0x%p "
 916                                     "for device %s\n",
 917                                     memhdr->addr, dev_info.device->name);
 918                 }
 919         }
 920 }
 921 #endif
 922
 923 static unsigned int
 924 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 925             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 926 {
 927         struct rte_pktmbuf_extmem *xmem;
 928         unsigned int ext_num, zone_num, elt_num;
 929         uint16_t elt_size;
 930
 931         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 932         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 933         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 934
 935         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 936         if (xmem == NULL) {
 937                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 938                                  "external buffer descriptors\n");
 939                 *ext_mem = NULL;
 940                 return 0;
 941         }
 942         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 943                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 944                 const struct rte_memzone *mz;
 945                 char mz_name[RTE_MEMZONE_NAMESIZE];
 946                 int ret;
 947
 948                 ret = snprintf(mz_name, sizeof(mz_name),
 949                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 950                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 951                         errno = ENAMETOOLONG;
 952                         ext_num = 0;
 953                         break;
 954                 }
 955                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 956                                                  socket_id,
 957                                                  RTE_MEMZONE_IOVA_CONTIG |
 958                                                  RTE_MEMZONE_1GB |
 959                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 960                                                  EXTBUF_ZONE_SIZE);
 961                 if (mz == NULL) {
 962                         /*
 963                          * The caller exits on external buffer creation
 964                          * error, so there is no need to free memzones.
 965                          */
 966                         errno = ENOMEM;
 967                         ext_num = 0;
 968                         break;
 969                 }
 970                 xseg->buf_ptr = mz->addr;
 971                 xseg->buf_iova = mz->iova;
 972                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 973                 xseg->elt_size = elt_size;
 974         }
 975         if (ext_num == 0 && xmem != NULL) {
 976                 free(xmem);
 977                 xmem = NULL;
 978         }
 979         *ext_mem = xmem;
 980         return ext_num;
 981 }
 982
 983 /*
 984  * Configuration initialisation done once at init time.
 985  */
 986 static struct rte_mempool *
 987 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 988                  unsigned int socket_id, uint16_t size_idx)
 989 {
 990         char pool_name[RTE_MEMPOOL_NAMESIZE];
 991         struct rte_mempool *rte_mp = NULL;
 992 #ifndef RTE_EXEC_ENV_WINDOWS
 993         uint32_t mb_size;
 994
 995         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 996 #endif
 997         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 998
 999         TESTPMD_LOG(INFO,
1000                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1001                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1002
1003         switch (mp_alloc_type) {
1004         case MP_ALLOC_NATIVE:
1005                 {
1006                         /* wrapper to rte_mempool_create() */
1007                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1008                                         rte_mbuf_best_mempool_ops());
1009                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1010                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1011                         break;
1012                 }
1013 #ifndef RTE_EXEC_ENV_WINDOWS
1014         case MP_ALLOC_ANON:
1015                 {
1016                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1017                                 mb_size, (unsigned int) mb_mempool_cache,
1018                                 sizeof(struct rte_pktmbuf_pool_private),
1019                                 socket_id, mempool_flags);
1020                         if (rte_mp == NULL)
1021                                 goto err;
1022
1023                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1024                                 rte_mempool_free(rte_mp);
1025                                 rte_mp = NULL;
1026                                 goto err;
1027                         }
1028                         rte_pktmbuf_pool_init(rte_mp, NULL);
1029                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1030                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1031                         break;
1032                 }
1033         case MP_ALLOC_XMEM:
1034         case MP_ALLOC_XMEM_HUGE:
1035                 {
1036                         int heap_socket;
1037                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1038
1039                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1040                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1041
1042                         heap_socket =
1043                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1044                         if (heap_socket < 0)
1045                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1046
1047                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1048                                         rte_mbuf_best_mempool_ops());
1049                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1050                                         mb_mempool_cache, 0, mbuf_seg_size,
1051                                         heap_socket);
1052                         break;
1053                 }
1054 #endif
1055         case MP_ALLOC_XBUF:
1056                 {
1057                         struct rte_pktmbuf_extmem *ext_mem;
1058                         unsigned int ext_num;
1059
1060                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1061                                                socket_id, pool_name, &ext_mem);
1062                         if (ext_num == 0)
1063                                 rte_exit(EXIT_FAILURE,
1064                                          "Can't create pinned data buffers\n");
1065
1066                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1067                                         rte_mbuf_best_mempool_ops());
1068                         rte_mp = rte_pktmbuf_pool_create_extbuf
1069                                         (pool_name, nb_mbuf, mb_mempool_cache,
1070                                          0, mbuf_seg_size, socket_id,
1071                                          ext_mem, ext_num);
1072                         free(ext_mem);
1073                         break;
1074                 }
1075         default:
1076                 {
1077                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1078                 }
1079         }
1080
1081 #ifndef RTE_EXEC_ENV_WINDOWS
1082 err:
1083 #endif
1084         if (rte_mp == NULL) {
1085                 rte_exit(EXIT_FAILURE,
1086                         "Creation of mbuf pool for socket %u failed: %s\n",
1087                         socket_id, rte_strerror(rte_errno));
1088         } else if (verbose_level > 0) {
1089                 rte_mempool_dump(stdout, rte_mp);
1090         }
1091         return rte_mp;
1092 }
1093
1094 /*
1095  * Check given socket id is valid or not with NUMA mode,
1096  * if valid, return 0, else return -1
1097  */
1098 static int
1099 check_socket_id(const unsigned int socket_id)
1100 {
1101         static int warning_once = 0;
1102
1103         if (new_socket_id(socket_id)) {
1104                 if (!warning_once && numa_support)
1105                         fprintf(stderr,
1106                                 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1107                 warning_once = 1;
1108                 return -1;
1109         }
1110         return 0;
1111 }
1112
1113 /*
1114  * Get the allowed maximum number of RX queues.
1115  * *pid return the port id which has minimal value of
1116  * max_rx_queues in all ports.
1117  */
1118 queueid_t
1119 get_allowed_max_nb_rxq(portid_t *pid)
1120 {
1121         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1122         bool max_rxq_valid = false;
1123         portid_t pi;
1124         struct rte_eth_dev_info dev_info;
1125
1126         RTE_ETH_FOREACH_DEV(pi) {
1127                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1128                         continue;
1129
1130                 max_rxq_valid = true;
1131                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1132                         allowed_max_rxq = dev_info.max_rx_queues;
1133                         *pid = pi;
1134                 }
1135         }
1136         return max_rxq_valid ? allowed_max_rxq : 0;
1137 }
1138
1139 /*
1140  * Check input rxq is valid or not.
1141  * If input rxq is not greater than any of maximum number
1142  * of RX queues of all ports, it is valid.
1143  * if valid, return 0, else return -1
1144  */
1145 int
1146 check_nb_rxq(queueid_t rxq)
1147 {
1148         queueid_t allowed_max_rxq;
1149         portid_t pid = 0;
1150
1151         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1152         if (rxq > allowed_max_rxq) {
1153                 fprintf(stderr,
1154                         "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1155                         rxq, allowed_max_rxq, pid);
1156                 return -1;
1157         }
1158         return 0;
1159 }
1160
1161 /*
1162  * Get the allowed maximum number of TX queues.
1163  * *pid return the port id which has minimal value of
1164  * max_tx_queues in all ports.
1165  */
1166 queueid_t
1167 get_allowed_max_nb_txq(portid_t *pid)
1168 {
1169         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1170         bool max_txq_valid = false;
1171         portid_t pi;
1172         struct rte_eth_dev_info dev_info;
1173
1174         RTE_ETH_FOREACH_DEV(pi) {
1175                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1176                         continue;
1177
1178                 max_txq_valid = true;
1179                 if (dev_info.max_tx_queues < allowed_max_txq) {
1180                         allowed_max_txq = dev_info.max_tx_queues;
1181                         *pid = pi;
1182                 }
1183         }
1184         return max_txq_valid ? allowed_max_txq : 0;
1185 }
1186
1187 /*
1188  * Check input txq is valid or not.
1189  * If input txq is not greater than any of maximum number
1190  * of TX queues of all ports, it is valid.
1191  * if valid, return 0, else return -1
1192  */
1193 int
1194 check_nb_txq(queueid_t txq)
1195 {
1196         queueid_t allowed_max_txq;
1197         portid_t pid = 0;
1198
1199         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1200         if (txq > allowed_max_txq) {
1201                 fprintf(stderr,
1202                         "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1203                         txq, allowed_max_txq, pid);
1204                 return -1;
1205         }
1206         return 0;
1207 }
1208
1209 /*
1210  * Get the allowed maximum number of RXDs of every rx queue.
1211  * *pid return the port id which has minimal value of
1212  * max_rxd in all queues of all ports.
1213  */
1214 static uint16_t
1215 get_allowed_max_nb_rxd(portid_t *pid)
1216 {
1217         uint16_t allowed_max_rxd = UINT16_MAX;
1218         portid_t pi;
1219         struct rte_eth_dev_info dev_info;
1220
1221         RTE_ETH_FOREACH_DEV(pi) {
1222                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1223                         continue;
1224
1225                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1226                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1227                         *pid = pi;
1228                 }
1229         }
1230         return allowed_max_rxd;
1231 }
1232
1233 /*
1234  * Get the allowed minimal number of RXDs of every rx queue.
1235  * *pid return the port id which has minimal value of
1236  * min_rxd in all queues of all ports.
1237  */
1238 static uint16_t
1239 get_allowed_min_nb_rxd(portid_t *pid)
1240 {
1241         uint16_t allowed_min_rxd = 0;
1242         portid_t pi;
1243         struct rte_eth_dev_info dev_info;
1244
1245         RTE_ETH_FOREACH_DEV(pi) {
1246                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1247                         continue;
1248
1249                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1250                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1251                         *pid = pi;
1252                 }
1253         }
1254
1255         return allowed_min_rxd;
1256 }
1257
1258 /*
1259  * Check input rxd is valid or not.
1260  * If input rxd is not greater than any of maximum number
1261  * of RXDs of every Rx queues and is not less than any of
1262  * minimal number of RXDs of every Rx queues, it is valid.
1263  * if valid, return 0, else return -1
1264  */
1265 int
1266 check_nb_rxd(queueid_t rxd)
1267 {
1268         uint16_t allowed_max_rxd;
1269         uint16_t allowed_min_rxd;
1270         portid_t pid = 0;
1271
1272         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1273         if (rxd > allowed_max_rxd) {
1274                 fprintf(stderr,
1275                         "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1276                         rxd, allowed_max_rxd, pid);
1277                 return -1;
1278         }
1279
1280         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1281         if (rxd < allowed_min_rxd) {
1282                 fprintf(stderr,
1283                         "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1284                         rxd, allowed_min_rxd, pid);
1285                 return -1;
1286         }
1287
1288         return 0;
1289 }
1290
1291 /*
1292  * Get the allowed maximum number of TXDs of every rx queues.
1293  * *pid return the port id which has minimal value of
1294  * max_txd in every tx queue.
1295  */
1296 static uint16_t
1297 get_allowed_max_nb_txd(portid_t *pid)
1298 {
1299         uint16_t allowed_max_txd = UINT16_MAX;
1300         portid_t pi;
1301         struct rte_eth_dev_info dev_info;
1302
1303         RTE_ETH_FOREACH_DEV(pi) {
1304                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1305                         continue;
1306
1307                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1308                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1309                         *pid = pi;
1310                 }
1311         }
1312         return allowed_max_txd;
1313 }
1314
1315 /*
1316  * Get the allowed maximum number of TXDs of every tx queues.
1317  * *pid return the port id which has minimal value of
1318  * min_txd in every tx queue.
1319  */
1320 static uint16_t
1321 get_allowed_min_nb_txd(portid_t *pid)
1322 {
1323         uint16_t allowed_min_txd = 0;
1324         portid_t pi;
1325         struct rte_eth_dev_info dev_info;
1326
1327         RTE_ETH_FOREACH_DEV(pi) {
1328                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1329                         continue;
1330
1331                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1332                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1333                         *pid = pi;
1334                 }
1335         }
1336
1337         return allowed_min_txd;
1338 }
1339
1340 /*
1341  * Check input txd is valid or not.
1342  * If input txd is not greater than any of maximum number
1343  * of TXDs of every Rx queues, it is valid.
1344  * if valid, return 0, else return -1
1345  */
1346 int
1347 check_nb_txd(queueid_t txd)
1348 {
1349         uint16_t allowed_max_txd;
1350         uint16_t allowed_min_txd;
1351         portid_t pid = 0;
1352
1353         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1354         if (txd > allowed_max_txd) {
1355                 fprintf(stderr,
1356                         "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1357                         txd, allowed_max_txd, pid);
1358                 return -1;
1359         }
1360
1361         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1362         if (txd < allowed_min_txd) {
1363                 fprintf(stderr,
1364                         "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1365                         txd, allowed_min_txd, pid);
1366                 return -1;
1367         }
1368         return 0;
1369 }
1370
1371
1372 /*
1373  * Get the allowed maximum number of hairpin queues.
1374  * *pid return the port id which has minimal value of
1375  * max_hairpin_queues in all ports.
1376  */
1377 queueid_t
1378 get_allowed_max_nb_hairpinq(portid_t *pid)
1379 {
1380         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1381         portid_t pi;
1382         struct rte_eth_hairpin_cap cap;
1383
1384         RTE_ETH_FOREACH_DEV(pi) {
1385                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1386                         *pid = pi;
1387                         return 0;
1388                 }
1389                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1390                         allowed_max_hairpinq = cap.max_nb_queues;
1391                         *pid = pi;
1392                 }
1393         }
1394         return allowed_max_hairpinq;
1395 }
1396
1397 /*
1398  * Check input hairpin is valid or not.
1399  * If input hairpin is not greater than any of maximum number
1400  * of hairpin queues of all ports, it is valid.
1401  * if valid, return 0, else return -1
1402  */
1403 int
1404 check_nb_hairpinq(queueid_t hairpinq)
1405 {
1406         queueid_t allowed_max_hairpinq;
1407         portid_t pid = 0;
1408
1409         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1410         if (hairpinq > allowed_max_hairpinq) {
1411                 fprintf(stderr,
1412                         "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1413                         hairpinq, allowed_max_hairpinq, pid);
1414                 return -1;
1415         }
1416         return 0;
1417 }
1418
1419 static void
1420 init_config_port_offloads(portid_t pid, uint32_t socket_id)
1421 {
1422         struct rte_port *port = &ports[pid];
1423         uint16_t data_size;
1424         int ret;
1425         int i;
1426
1427         port->dev_conf.txmode = tx_mode;
1428         port->dev_conf.rxmode = rx_mode;
1429
1430         ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1431         if (ret != 0)
1432                 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1433
1434         ret = update_jumbo_frame_offload(pid);
1435         if (ret != 0)
1436                 fprintf(stderr,
1437                         "Updating jumbo frame offload failed for port %u\n",
1438                         pid);
1439
1440         if (!(port->dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1441                 port->dev_conf.txmode.offloads &=
1442                         ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1443
1444         /* Apply Rx offloads configuration */
1445         for (i = 0; i < port->dev_info.max_rx_queues; i++)
1446                 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1447         /* Apply Tx offloads configuration */
1448         for (i = 0; i < port->dev_info.max_tx_queues; i++)
1449                 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1450
1451         if (eth_link_speed)
1452                 port->dev_conf.link_speeds = eth_link_speed;
1453
1454         /* set flag to initialize port/queue */
1455         port->need_reconfig = 1;
1456         port->need_reconfig_queues = 1;
1457         port->socket_id = socket_id;
1458         port->tx_metadata = 0;
1459
1460         /*
1461          * Check for maximum number of segments per MTU.
1462          * Accordingly update the mbuf data size.
1463          */
1464         if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1465             port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1466                 data_size = rx_mode.max_rx_pkt_len /
1467                         port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1468
1469                 if ((data_size + RTE_PKTMBUF_HEADROOM) > mbuf_data_size[0]) {
1470                         mbuf_data_size[0] = data_size + RTE_PKTMBUF_HEADROOM;
1471                         TESTPMD_LOG(WARNING,
1472                                     "Configured mbuf size of the first segment %hu\n",
1473                                     mbuf_data_size[0]);
1474                 }
1475         }
1476 }
1477
1478 static void
1479 init_config(void)
1480 {
1481         portid_t pid;
1482         struct rte_mempool *mbp;
1483         unsigned int nb_mbuf_per_pool;
1484         lcoreid_t  lc_id;
1485         struct rte_gro_param gro_param;
1486         uint32_t gso_types;
1487
1488         /* Configuration of logical cores. */
1489         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1490                                 sizeof(struct fwd_lcore *) * nb_lcores,
1491                                 RTE_CACHE_LINE_SIZE);
1492         if (fwd_lcores == NULL) {
1493                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1494                                                         "failed\n", nb_lcores);
1495         }
1496         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1497                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1498                                                sizeof(struct fwd_lcore),
1499                                                RTE_CACHE_LINE_SIZE);
1500                 if (fwd_lcores[lc_id] == NULL) {
1501                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1502                                                                 "failed\n");
1503                 }
1504                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1505         }
1506
1507         RTE_ETH_FOREACH_DEV(pid) {
1508                 uint32_t socket_id;
1509
1510                 if (numa_support) {
1511                         socket_id = port_numa[pid];
1512                         if (port_numa[pid] == NUMA_NO_CONFIG) {
1513                                 socket_id = rte_eth_dev_socket_id(pid);
1514
1515                                 /*
1516                                  * if socket_id is invalid,
1517                                  * set to the first available socket.
1518                                  */
1519                                 if (check_socket_id(socket_id) < 0)
1520                                         socket_id = socket_ids[0];
1521                         }
1522                 } else {
1523                         socket_id = (socket_num == UMA_NO_CONFIG) ?
1524                                     0 : socket_num;
1525                 }
1526                 /* Apply default TxRx configuration for all ports */
1527                 init_config_port_offloads(pid, socket_id);
1528         }
1529         /*
1530          * Create pools of mbuf.
1531          * If NUMA support is disabled, create a single pool of mbuf in
1532          * socket 0 memory by default.
1533          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1534          *
1535          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1536          * nb_txd can be configured at run time.
1537          */
1538         if (param_total_num_mbufs)
1539                 nb_mbuf_per_pool = param_total_num_mbufs;
1540         else {
1541                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1542                         (nb_lcores * mb_mempool_cache) +
1543                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1544                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1545         }
1546
1547         if (numa_support) {
1548                 uint8_t i, j;
1549
1550                 for (i = 0; i < num_sockets; i++)
1551                         for (j = 0; j < mbuf_data_size_n; j++)
1552                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1553                                         mbuf_pool_create(mbuf_data_size[j],
1554                                                           nb_mbuf_per_pool,
1555                                                           socket_ids[i], j);
1556         } else {
1557                 uint8_t i;
1558
1559                 for (i = 0; i < mbuf_data_size_n; i++)
1560                         mempools[i] = mbuf_pool_create
1561                                         (mbuf_data_size[i],
1562                                          nb_mbuf_per_pool,
1563                                          socket_num == UMA_NO_CONFIG ?
1564                                          0 : socket_num, i);
1565         }
1566
1567         init_port_config();
1568
1569         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1570                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1571         /*
1572          * Records which Mbuf pool to use by each logical core, if needed.
1573          */
1574         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1575                 mbp = mbuf_pool_find(
1576                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1577
1578                 if (mbp == NULL)
1579                         mbp = mbuf_pool_find(0, 0);
1580                 fwd_lcores[lc_id]->mbp = mbp;
1581                 /* initialize GSO context */
1582                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1583                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1584                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1585                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1586                         RTE_ETHER_CRC_LEN;
1587                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1588         }
1589
1590         fwd_config_setup();
1591
1592         /* create a gro context for each lcore */
1593         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1594         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1595         gro_param.max_item_per_flow = MAX_PKT_BURST;
1596         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1597                 gro_param.socket_id = rte_lcore_to_socket_id(
1598                                 fwd_lcores_cpuids[lc_id]);
1599                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1600                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1601                         rte_exit(EXIT_FAILURE,
1602                                         "rte_gro_ctx_create() failed\n");
1603                 }
1604         }
1605 }
1606
1607
1608 void
1609 reconfig(portid_t new_port_id, unsigned socket_id)
1610 {
1611         /* Reconfiguration of Ethernet ports. */
1612         init_config_port_offloads(new_port_id, socket_id);
1613         init_port_config();
1614 }
1615
1616
1617 int
1618 init_fwd_streams(void)
1619 {
1620         portid_t pid;
1621         struct rte_port *port;
1622         streamid_t sm_id, nb_fwd_streams_new;
1623         queueid_t q;
1624
1625         /* set socket id according to numa or not */
1626         RTE_ETH_FOREACH_DEV(pid) {
1627                 port = &ports[pid];
1628                 if (nb_rxq > port->dev_info.max_rx_queues) {
1629                         fprintf(stderr,
1630                                 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1631                                 nb_rxq, port->dev_info.max_rx_queues);
1632                         return -1;
1633                 }
1634                 if (nb_txq > port->dev_info.max_tx_queues) {
1635                         fprintf(stderr,
1636                                 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1637                                 nb_txq, port->dev_info.max_tx_queues);
1638                         return -1;
1639                 }
1640                 if (numa_support) {
1641                         if (port_numa[pid] != NUMA_NO_CONFIG)
1642                                 port->socket_id = port_numa[pid];
1643                         else {
1644                                 port->socket_id = rte_eth_dev_socket_id(pid);
1645
1646                                 /*
1647                                  * if socket_id is invalid,
1648                                  * set to the first available socket.
1649                                  */
1650                                 if (check_socket_id(port->socket_id) < 0)
1651                                         port->socket_id = socket_ids[0];
1652                         }
1653                 }
1654                 else {
1655                         if (socket_num == UMA_NO_CONFIG)
1656                                 port->socket_id = 0;
1657                         else
1658                                 port->socket_id = socket_num;
1659                 }
1660         }
1661
1662         q = RTE_MAX(nb_rxq, nb_txq);
1663         if (q == 0) {
1664                 fprintf(stderr,
1665                         "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1666                 return -1;
1667         }
1668         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1669         if (nb_fwd_streams_new == nb_fwd_streams)
1670                 return 0;
1671         /* clear the old */
1672         if (fwd_streams != NULL) {
1673                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1674                         if (fwd_streams[sm_id] == NULL)
1675                                 continue;
1676                         rte_free(fwd_streams[sm_id]);
1677                         fwd_streams[sm_id] = NULL;
1678                 }
1679                 rte_free(fwd_streams);
1680                 fwd_streams = NULL;
1681         }
1682
1683         /* init new */
1684         nb_fwd_streams = nb_fwd_streams_new;
1685         if (nb_fwd_streams) {
1686                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1687                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1688                         RTE_CACHE_LINE_SIZE);
1689                 if (fwd_streams == NULL)
1690                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1691                                  " (struct fwd_stream *)) failed\n",
1692                                  nb_fwd_streams);
1693
1694                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1695                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1696                                 " struct fwd_stream", sizeof(struct fwd_stream),
1697                                 RTE_CACHE_LINE_SIZE);
1698                         if (fwd_streams[sm_id] == NULL)
1699                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1700                                          "(struct fwd_stream) failed\n");
1701                 }
1702         }
1703
1704         return 0;
1705 }
1706
1707 static void
1708 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1709 {
1710         uint64_t total_burst, sburst;
1711         uint64_t nb_burst;
1712         uint64_t burst_stats[4];
1713         uint16_t pktnb_stats[4];
1714         uint16_t nb_pkt;
1715         int burst_percent[4], sburstp;
1716         int i;
1717
1718         /*
1719          * First compute the total number of packet bursts and the
1720          * two highest numbers of bursts of the same number of packets.
1721          */
1722         memset(&burst_stats, 0x0, sizeof(burst_stats));
1723         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1724
1725         /* Show stats for 0 burst size always */
1726         total_burst = pbs->pkt_burst_spread[0];
1727         burst_stats[0] = pbs->pkt_burst_spread[0];
1728         pktnb_stats[0] = 0;
1729
1730         /* Find the next 2 burst sizes with highest occurrences. */
1731         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1732                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1733
1734                 if (nb_burst == 0)
1735                         continue;
1736
1737                 total_burst += nb_burst;
1738
1739                 if (nb_burst > burst_stats[1]) {
1740                         burst_stats[2] = burst_stats[1];
1741                         pktnb_stats[2] = pktnb_stats[1];
1742                         burst_stats[1] = nb_burst;
1743                         pktnb_stats[1] = nb_pkt;
1744                 } else if (nb_burst > burst_stats[2]) {
1745                         burst_stats[2] = nb_burst;
1746                         pktnb_stats[2] = nb_pkt;
1747                 }
1748         }
1749         if (total_burst == 0)
1750                 return;
1751
1752         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1753         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1754                 if (i == 3) {
1755                         printf("%d%% of other]\n", 100 - sburstp);
1756                         return;
1757                 }
1758
1759                 sburst += burst_stats[i];
1760                 if (sburst == total_burst) {
1761                         printf("%d%% of %d pkts]\n",
1762                                 100 - sburstp, (int) pktnb_stats[i]);
1763                         return;
1764                 }
1765
1766                 burst_percent[i] =
1767                         (double)burst_stats[i] / total_burst * 100;
1768                 printf("%d%% of %d pkts + ",
1769                         burst_percent[i], (int) pktnb_stats[i]);
1770                 sburstp += burst_percent[i];
1771         }
1772 }
1773
1774 static void
1775 fwd_stream_stats_display(streamid_t stream_id)
1776 {
1777         struct fwd_stream *fs;
1778         static const char *fwd_top_stats_border = "-------";
1779
1780         fs = fwd_streams[stream_id];
1781         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1782             (fs->fwd_dropped == 0))
1783                 return;
1784         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1785                "TX Port=%2d/Queue=%2d %s\n",
1786                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1787                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1788         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1789                " TX-dropped: %-14"PRIu64,
1790                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1791
1792         /* if checksum mode */
1793         if (cur_fwd_eng == &csum_fwd_engine) {
1794                 printf("  RX- bad IP checksum: %-14"PRIu64
1795                        "  Rx- bad L4 checksum: %-14"PRIu64
1796                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1797                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1798                         fs->rx_bad_outer_l4_csum);
1799                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1800                         fs->rx_bad_outer_ip_csum);
1801         } else {
1802                 printf("\n");
1803         }
1804
1805         if (record_burst_stats) {
1806                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1807                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1808         }
1809 }
1810
1811 void
1812 fwd_stats_display(void)
1813 {
1814         static const char *fwd_stats_border = "----------------------";
1815         static const char *acc_stats_border = "+++++++++++++++";
1816         struct {
1817                 struct fwd_stream *rx_stream;
1818                 struct fwd_stream *tx_stream;
1819                 uint64_t tx_dropped;
1820                 uint64_t rx_bad_ip_csum;
1821                 uint64_t rx_bad_l4_csum;
1822                 uint64_t rx_bad_outer_l4_csum;
1823                 uint64_t rx_bad_outer_ip_csum;
1824         } ports_stats[RTE_MAX_ETHPORTS];
1825         uint64_t total_rx_dropped = 0;
1826         uint64_t total_tx_dropped = 0;
1827         uint64_t total_rx_nombuf = 0;
1828         struct rte_eth_stats stats;
1829         uint64_t fwd_cycles = 0;
1830         uint64_t total_recv = 0;
1831         uint64_t total_xmit = 0;
1832         struct rte_port *port;
1833         streamid_t sm_id;
1834         portid_t pt_id;
1835         int i;
1836
1837         memset(ports_stats, 0, sizeof(ports_stats));
1838
1839         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1840                 struct fwd_stream *fs = fwd_streams[sm_id];
1841
1842                 if (cur_fwd_config.nb_fwd_streams >
1843                     cur_fwd_config.nb_fwd_ports) {
1844                         fwd_stream_stats_display(sm_id);
1845                 } else {
1846                         ports_stats[fs->tx_port].tx_stream = fs;
1847                         ports_stats[fs->rx_port].rx_stream = fs;
1848                 }
1849
1850                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1851
1852                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1853                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1854                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1855                                 fs->rx_bad_outer_l4_csum;
1856                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1857                                 fs->rx_bad_outer_ip_csum;
1858
1859                 if (record_core_cycles)
1860                         fwd_cycles += fs->core_cycles;
1861         }
1862         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1863                 pt_id = fwd_ports_ids[i];
1864                 port = &ports[pt_id];
1865
1866                 rte_eth_stats_get(pt_id, &stats);
1867                 stats.ipackets -= port->stats.ipackets;
1868                 stats.opackets -= port->stats.opackets;
1869                 stats.ibytes -= port->stats.ibytes;
1870                 stats.obytes -= port->stats.obytes;
1871                 stats.imissed -= port->stats.imissed;
1872                 stats.oerrors -= port->stats.oerrors;
1873                 stats.rx_nombuf -= port->stats.rx_nombuf;
1874
1875                 total_recv += stats.ipackets;
1876                 total_xmit += stats.opackets;
1877                 total_rx_dropped += stats.imissed;
1878                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1879                 total_tx_dropped += stats.oerrors;
1880                 total_rx_nombuf  += stats.rx_nombuf;
1881
1882                 printf("\n  %s Forward statistics for port %-2d %s\n",
1883                        fwd_stats_border, pt_id, fwd_stats_border);
1884
1885                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1886                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1887                        stats.ipackets + stats.imissed);
1888
1889                 if (cur_fwd_eng == &csum_fwd_engine) {
1890                         printf("  Bad-ipcsum: %-14"PRIu64
1891                                " Bad-l4csum: %-14"PRIu64
1892                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1893                                ports_stats[pt_id].rx_bad_ip_csum,
1894                                ports_stats[pt_id].rx_bad_l4_csum,
1895                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1896                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1897                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1898                 }
1899                 if (stats.ierrors + stats.rx_nombuf > 0) {
1900                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1901                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1902                 }
1903
1904                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1905                        "TX-total: %-"PRIu64"\n",
1906                        stats.opackets, ports_stats[pt_id].tx_dropped,
1907                        stats.opackets + ports_stats[pt_id].tx_dropped);
1908
1909                 if (record_burst_stats) {
1910                         if (ports_stats[pt_id].rx_stream)
1911                                 pkt_burst_stats_display("RX",
1912                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1913                         if (ports_stats[pt_id].tx_stream)
1914                                 pkt_burst_stats_display("TX",
1915                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1916                 }
1917
1918                 printf("  %s--------------------------------%s\n",
1919                        fwd_stats_border, fwd_stats_border);
1920         }
1921
1922         printf("\n  %s Accumulated forward statistics for all ports"
1923                "%s\n",
1924                acc_stats_border, acc_stats_border);
1925         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1926                "%-"PRIu64"\n"
1927                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1928                "%-"PRIu64"\n",
1929                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1930                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1931         if (total_rx_nombuf > 0)
1932                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1933         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1934                "%s\n",
1935                acc_stats_border, acc_stats_border);
1936         if (record_core_cycles) {
1937 #define CYC_PER_MHZ 1E6
1938                 if (total_recv > 0 || total_xmit > 0) {
1939                         uint64_t total_pkts = 0;
1940                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1941                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1942                                 total_pkts = total_xmit;
1943                         else
1944                                 total_pkts = total_recv;
1945
1946                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1947                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1948                                " MHz Clock\n",
1949                                (double) fwd_cycles / total_pkts,
1950                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1951                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1952                 }
1953         }
1954 }
1955
1956 void
1957 fwd_stats_reset(void)
1958 {
1959         streamid_t sm_id;
1960         portid_t pt_id;
1961         int i;
1962
1963         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1964                 pt_id = fwd_ports_ids[i];
1965                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1966         }
1967         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1968                 struct fwd_stream *fs = fwd_streams[sm_id];
1969
1970                 fs->rx_packets = 0;
1971                 fs->tx_packets = 0;
1972                 fs->fwd_dropped = 0;
1973                 fs->rx_bad_ip_csum = 0;
1974                 fs->rx_bad_l4_csum = 0;
1975                 fs->rx_bad_outer_l4_csum = 0;
1976                 fs->rx_bad_outer_ip_csum = 0;
1977
1978                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1979                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1980                 fs->core_cycles = 0;
1981         }
1982 }
1983
1984 static void
1985 flush_fwd_rx_queues(void)
1986 {
1987         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1988         portid_t  rxp;
1989         portid_t port_id;
1990         queueid_t rxq;
1991         uint16_t  nb_rx;
1992         uint16_t  i;
1993         uint8_t   j;
1994         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1995         uint64_t timer_period;
1996
1997         /* convert to number of cycles */
1998         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1999
2000         for (j = 0; j < 2; j++) {
2001                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2002                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2003                                 port_id = fwd_ports_ids[rxp];
2004                                 /**
2005                                 * testpmd can stuck in the below do while loop
2006                                 * if rte_eth_rx_burst() always returns nonzero
2007                                 * packets. So timer is added to exit this loop
2008                                 * after 1sec timer expiry.
2009                                 */
2010                                 prev_tsc = rte_rdtsc();
2011                                 do {
2012                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2013                                                 pkts_burst, MAX_PKT_BURST);
2014                                         for (i = 0; i < nb_rx; i++)
2015                                                 rte_pktmbuf_free(pkts_burst[i]);
2016
2017                                         cur_tsc = rte_rdtsc();
2018                                         diff_tsc = cur_tsc - prev_tsc;
2019                                         timer_tsc += diff_tsc;
2020                                 } while ((nb_rx > 0) &&
2021                                         (timer_tsc < timer_period));
2022                                 timer_tsc = 0;
2023                         }
2024                 }
2025                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2026         }
2027 }
2028
2029 static void
2030 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2031 {
2032         struct fwd_stream **fsm;
2033         streamid_t nb_fs;
2034         streamid_t sm_id;
2035 #ifdef RTE_LIB_BITRATESTATS
2036         uint64_t tics_per_1sec;
2037         uint64_t tics_datum;
2038         uint64_t tics_current;
2039         uint16_t i, cnt_ports;
2040
2041         cnt_ports = nb_ports;
2042         tics_datum = rte_rdtsc();
2043         tics_per_1sec = rte_get_timer_hz();
2044 #endif
2045         fsm = &fwd_streams[fc->stream_idx];
2046         nb_fs = fc->stream_nb;
2047         do {
2048                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2049                         (*pkt_fwd)(fsm[sm_id]);
2050 #ifdef RTE_LIB_BITRATESTATS
2051                 if (bitrate_enabled != 0 &&
2052                                 bitrate_lcore_id == rte_lcore_id()) {
2053                         tics_current = rte_rdtsc();
2054                         if (tics_current - tics_datum >= tics_per_1sec) {
2055                                 /* Periodic bitrate calculation */
2056                                 for (i = 0; i < cnt_ports; i++)
2057                                         rte_stats_bitrate_calc(bitrate_data,
2058                                                 ports_ids[i]);
2059                                 tics_datum = tics_current;
2060                         }
2061                 }
2062 #endif
2063 #ifdef RTE_LIB_LATENCYSTATS
2064                 if (latencystats_enabled != 0 &&
2065                                 latencystats_lcore_id == rte_lcore_id())
2066                         rte_latencystats_update();
2067 #endif
2068
2069         } while (! fc->stopped);
2070 }
2071
2072 static int
2073 start_pkt_forward_on_core(void *fwd_arg)
2074 {
2075         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2076                              cur_fwd_config.fwd_eng->packet_fwd);
2077         return 0;
2078 }
2079
2080 /*
2081  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2082  * Used to start communication flows in network loopback test configurations.
2083  */
2084 static int
2085 run_one_txonly_burst_on_core(void *fwd_arg)
2086 {
2087         struct fwd_lcore *fwd_lc;
2088         struct fwd_lcore tmp_lcore;
2089
2090         fwd_lc = (struct fwd_lcore *) fwd_arg;
2091         tmp_lcore = *fwd_lc;
2092         tmp_lcore.stopped = 1;
2093         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2094         return 0;
2095 }
2096
2097 /*
2098  * Launch packet forwarding:
2099  *     - Setup per-port forwarding context.
2100  *     - launch logical cores with their forwarding configuration.
2101  */
2102 static void
2103 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2104 {
2105         port_fwd_begin_t port_fwd_begin;
2106         unsigned int i;
2107         unsigned int lc_id;
2108         int diag;
2109
2110         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2111         if (port_fwd_begin != NULL) {
2112                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2113                         (*port_fwd_begin)(fwd_ports_ids[i]);
2114         }
2115         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2116                 lc_id = fwd_lcores_cpuids[i];
2117                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2118                         fwd_lcores[i]->stopped = 0;
2119                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2120                                                      fwd_lcores[i], lc_id);
2121                         if (diag != 0)
2122                                 fprintf(stderr,
2123                                         "launch lcore %u failed - diag=%d\n",
2124                                         lc_id, diag);
2125                 }
2126         }
2127 }
2128
2129 /*
2130  * Launch packet forwarding configuration.
2131  */
2132 void
2133 start_packet_forwarding(int with_tx_first)
2134 {
2135         port_fwd_begin_t port_fwd_begin;
2136         port_fwd_end_t  port_fwd_end;
2137         unsigned int i;
2138
2139         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2140                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2141
2142         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2143                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2144
2145         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2146                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2147                 (!nb_rxq || !nb_txq))
2148                 rte_exit(EXIT_FAILURE,
2149                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2150                         cur_fwd_eng->fwd_mode_name);
2151
2152         if (all_ports_started() == 0) {
2153                 fprintf(stderr, "Not all ports were started\n");
2154                 return;
2155         }
2156         if (test_done == 0) {
2157                 fprintf(stderr, "Packet forwarding already started\n");
2158                 return;
2159         }
2160         test_done = 0;
2161
2162         fwd_config_setup();
2163
2164         if(!no_flush_rx)
2165                 flush_fwd_rx_queues();
2166
2167         pkt_fwd_config_display(&cur_fwd_config);
2168         rxtx_config_display();
2169
2170         fwd_stats_reset();
2171         if (with_tx_first) {
2172                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2173                 if (port_fwd_begin != NULL) {
2174                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2175                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2176                 }
2177                 while (with_tx_first--) {
2178                         launch_packet_forwarding(
2179                                         run_one_txonly_burst_on_core);
2180                         rte_eal_mp_wait_lcore();
2181                 }
2182                 port_fwd_end = tx_only_engine.port_fwd_end;
2183                 if (port_fwd_end != NULL) {
2184                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2185                                 (*port_fwd_end)(fwd_ports_ids[i]);
2186                 }
2187         }
2188         launch_packet_forwarding(start_pkt_forward_on_core);
2189 }
2190
2191 void
2192 stop_packet_forwarding(void)
2193 {
2194         port_fwd_end_t port_fwd_end;
2195         lcoreid_t lc_id;
2196         portid_t pt_id;
2197         int i;
2198
2199         if (test_done) {
2200                 fprintf(stderr, "Packet forwarding not started\n");
2201                 return;
2202         }
2203         printf("Telling cores to stop...");
2204         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2205                 fwd_lcores[lc_id]->stopped = 1;
2206         printf("\nWaiting for lcores to finish...\n");
2207         rte_eal_mp_wait_lcore();
2208         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2209         if (port_fwd_end != NULL) {
2210                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2211                         pt_id = fwd_ports_ids[i];
2212                         (*port_fwd_end)(pt_id);
2213                 }
2214         }
2215
2216         fwd_stats_display();
2217
2218         printf("\nDone.\n");
2219         test_done = 1;
2220 }
2221
2222 void
2223 dev_set_link_up(portid_t pid)
2224 {
2225         if (rte_eth_dev_set_link_up(pid) < 0)
2226                 fprintf(stderr, "\nSet link up fail.\n");
2227 }
2228
2229 void
2230 dev_set_link_down(portid_t pid)
2231 {
2232         if (rte_eth_dev_set_link_down(pid) < 0)
2233                 fprintf(stderr, "\nSet link down fail.\n");
2234 }
2235
2236 static int
2237 all_ports_started(void)
2238 {
2239         portid_t pi;
2240         struct rte_port *port;
2241
2242         RTE_ETH_FOREACH_DEV(pi) {
2243                 port = &ports[pi];
2244                 /* Check if there is a port which is not started */
2245                 if ((port->port_status != RTE_PORT_STARTED) &&
2246                         (port->slave_flag == 0))
2247                         return 0;
2248         }
2249
2250         /* No port is not started */
2251         return 1;
2252 }
2253
2254 int
2255 port_is_stopped(portid_t port_id)
2256 {
2257         struct rte_port *port = &ports[port_id];
2258
2259         if ((port->port_status != RTE_PORT_STOPPED) &&
2260             (port->slave_flag == 0))
2261                 return 0;
2262         return 1;
2263 }
2264
2265 int
2266 all_ports_stopped(void)
2267 {
2268         portid_t pi;
2269
2270         RTE_ETH_FOREACH_DEV(pi) {
2271                 if (!port_is_stopped(pi))
2272                         return 0;
2273         }
2274
2275         return 1;
2276 }
2277
2278 int
2279 port_is_started(portid_t port_id)
2280 {
2281         if (port_id_is_invalid(port_id, ENABLED_WARN))
2282                 return 0;
2283
2284         if (ports[port_id].port_status != RTE_PORT_STARTED)
2285                 return 0;
2286
2287         return 1;
2288 }
2289
2290 /* Configure the Rx and Tx hairpin queues for the selected port. */
2291 static int
2292 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2293 {
2294         queueid_t qi;
2295         struct rte_eth_hairpin_conf hairpin_conf = {
2296                 .peer_count = 1,
2297         };
2298         int i;
2299         int diag;
2300         struct rte_port *port = &ports[pi];
2301         uint16_t peer_rx_port = pi;
2302         uint16_t peer_tx_port = pi;
2303         uint32_t manual = 1;
2304         uint32_t tx_exp = hairpin_mode & 0x10;
2305
2306         if (!(hairpin_mode & 0xf)) {
2307                 peer_rx_port = pi;
2308                 peer_tx_port = pi;
2309                 manual = 0;
2310         } else if (hairpin_mode & 0x1) {
2311                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2312                                                        RTE_ETH_DEV_NO_OWNER);
2313                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2314                         peer_tx_port = rte_eth_find_next_owned_by(0,
2315                                                 RTE_ETH_DEV_NO_OWNER);
2316                 if (p_pi != RTE_MAX_ETHPORTS) {
2317                         peer_rx_port = p_pi;
2318                 } else {
2319                         uint16_t next_pi;
2320
2321                         /* Last port will be the peer RX port of the first. */
2322                         RTE_ETH_FOREACH_DEV(next_pi)
2323                                 peer_rx_port = next_pi;
2324                 }
2325                 manual = 1;
2326         } else if (hairpin_mode & 0x2) {
2327                 if (cnt_pi & 0x1) {
2328                         peer_rx_port = p_pi;
2329                 } else {
2330                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2331                                                 RTE_ETH_DEV_NO_OWNER);
2332                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2333                                 peer_rx_port = pi;
2334                 }
2335                 peer_tx_port = peer_rx_port;
2336                 manual = 1;
2337         }
2338
2339         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2340                 hairpin_conf.peers[0].port = peer_rx_port;
2341                 hairpin_conf.peers[0].queue = i + nb_rxq;
2342                 hairpin_conf.manual_bind = !!manual;
2343                 hairpin_conf.tx_explicit = !!tx_exp;
2344                 diag = rte_eth_tx_hairpin_queue_setup
2345                         (pi, qi, nb_txd, &hairpin_conf);
2346                 i++;
2347                 if (diag == 0)
2348                         continue;
2349
2350                 /* Fail to setup rx queue, return */
2351                 if (rte_atomic16_cmpset(&(port->port_status),
2352                                         RTE_PORT_HANDLING,
2353                                         RTE_PORT_STOPPED) == 0)
2354                         fprintf(stderr,
2355                                 "Port %d can not be set back to stopped\n", pi);
2356                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2357                         pi);
2358                 /* try to reconfigure queues next time */
2359                 port->need_reconfig_queues = 1;
2360                 return -1;
2361         }
2362         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2363                 hairpin_conf.peers[0].port = peer_tx_port;
2364                 hairpin_conf.peers[0].queue = i + nb_txq;
2365                 hairpin_conf.manual_bind = !!manual;
2366                 hairpin_conf.tx_explicit = !!tx_exp;
2367                 diag = rte_eth_rx_hairpin_queue_setup
2368                         (pi, qi, nb_rxd, &hairpin_conf);
2369                 i++;
2370                 if (diag == 0)
2371                         continue;
2372
2373                 /* Fail to setup rx queue, return */
2374                 if (rte_atomic16_cmpset(&(port->port_status),
2375                                         RTE_PORT_HANDLING,
2376                                         RTE_PORT_STOPPED) == 0)
2377                         fprintf(stderr,
2378                                 "Port %d can not be set back to stopped\n", pi);
2379                 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2380                         pi);
2381                 /* try to reconfigure queues next time */
2382                 port->need_reconfig_queues = 1;
2383                 return -1;
2384         }
2385         return 0;
2386 }
2387
2388 /* Configure the Rx with optional split. */
2389 int
2390 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2391                uint16_t nb_rx_desc, unsigned int socket_id,
2392                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2393 {
2394         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2395         unsigned int i, mp_n;
2396         int ret;
2397
2398         if (rx_pkt_nb_segs <= 1 ||
2399             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2400                 rx_conf->rx_seg = NULL;
2401                 rx_conf->rx_nseg = 0;
2402                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2403                                              nb_rx_desc, socket_id,
2404                                              rx_conf, mp);
2405                 return ret;
2406         }
2407         for (i = 0; i < rx_pkt_nb_segs; i++) {
2408                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2409                 struct rte_mempool *mpx;
2410                 /*
2411                  * Use last valid pool for the segments with number
2412                  * exceeding the pool index.
2413                  */
2414                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2415                 mpx = mbuf_pool_find(socket_id, mp_n);
2416                 /* Handle zero as mbuf data buffer size. */
2417                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2418                                    rx_pkt_seg_lengths[i] :
2419                                    mbuf_data_size[mp_n];
2420                 rx_seg->offset = i < rx_pkt_nb_offs ?
2421                                    rx_pkt_seg_offsets[i] : 0;
2422                 rx_seg->mp = mpx ? mpx : mp;
2423         }
2424         rx_conf->rx_nseg = rx_pkt_nb_segs;
2425         rx_conf->rx_seg = rx_useg;
2426         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2427                                     socket_id, rx_conf, NULL);
2428         rx_conf->rx_seg = NULL;
2429         rx_conf->rx_nseg = 0;
2430         return ret;
2431 }
2432
2433 int
2434 start_port(portid_t pid)
2435 {
2436         int diag, need_check_link_status = -1;
2437         portid_t pi;
2438         portid_t p_pi = RTE_MAX_ETHPORTS;
2439         portid_t pl[RTE_MAX_ETHPORTS];
2440         portid_t peer_pl[RTE_MAX_ETHPORTS];
2441         uint16_t cnt_pi = 0;
2442         uint16_t cfg_pi = 0;
2443         int peer_pi;
2444         queueid_t qi;
2445         struct rte_port *port;
2446         struct rte_eth_hairpin_cap cap;
2447
2448         if (port_id_is_invalid(pid, ENABLED_WARN))
2449                 return 0;
2450
2451         RTE_ETH_FOREACH_DEV(pi) {
2452                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2453                         continue;
2454
2455                 need_check_link_status = 0;
2456                 port = &ports[pi];
2457                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2458                                                  RTE_PORT_HANDLING) == 0) {
2459                         fprintf(stderr, "Port %d is now not stopped\n", pi);
2460                         continue;
2461                 }
2462
2463                 if (port->need_reconfig > 0) {
2464                         port->need_reconfig = 0;
2465
2466                         if (flow_isolate_all) {
2467                                 int ret = port_flow_isolate(pi, 1);
2468                                 if (ret) {
2469                                         fprintf(stderr,
2470                                                 "Failed to apply isolated mode on port %d\n",
2471                                                 pi);
2472                                         return -1;
2473                                 }
2474                         }
2475                         configure_rxtx_dump_callbacks(0);
2476                         printf("Configuring Port %d (socket %u)\n", pi,
2477                                         port->socket_id);
2478                         if (nb_hairpinq > 0 &&
2479                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2480                                 fprintf(stderr,
2481                                         "Port %d doesn't support hairpin queues\n",
2482                                         pi);
2483                                 return -1;
2484                         }
2485                         /* configure port */
2486                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2487                                                      nb_txq + nb_hairpinq,
2488                                                      &(port->dev_conf));
2489                         if (diag != 0) {
2490                                 if (rte_atomic16_cmpset(&(port->port_status),
2491                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2492                                         fprintf(stderr,
2493                                                 "Port %d can not be set back to stopped\n",
2494                                                 pi);
2495                                 fprintf(stderr, "Fail to configure port %d\n",
2496                                         pi);
2497                                 /* try to reconfigure port next time */
2498                                 port->need_reconfig = 1;
2499                                 return -1;
2500                         }
2501                 }
2502                 if (port->need_reconfig_queues > 0) {
2503                         port->need_reconfig_queues = 0;
2504                         /* setup tx queues */
2505                         for (qi = 0; qi < nb_txq; qi++) {
2506                                 if ((numa_support) &&
2507                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2508                                         diag = rte_eth_tx_queue_setup(pi, qi,
2509                                                 port->nb_tx_desc[qi],
2510                                                 txring_numa[pi],
2511                                                 &(port->tx_conf[qi]));
2512                                 else
2513                                         diag = rte_eth_tx_queue_setup(pi, qi,
2514                                                 port->nb_tx_desc[qi],
2515                                                 port->socket_id,
2516                                                 &(port->tx_conf[qi]));
2517
2518                                 if (diag == 0)
2519                                         continue;
2520
2521                                 /* Fail to setup tx queue, return */
2522                                 if (rte_atomic16_cmpset(&(port->port_status),
2523                                                         RTE_PORT_HANDLING,
2524                                                         RTE_PORT_STOPPED) == 0)
2525                                         fprintf(stderr,
2526                                                 "Port %d can not be set back to stopped\n",
2527                                                 pi);
2528                                 fprintf(stderr,
2529                                         "Fail to configure port %d tx queues\n",
2530                                         pi);
2531                                 /* try to reconfigure queues next time */
2532                                 port->need_reconfig_queues = 1;
2533                                 return -1;
2534                         }
2535                         for (qi = 0; qi < nb_rxq; qi++) {
2536                                 /* setup rx queues */
2537                                 if ((numa_support) &&
2538                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2539                                         struct rte_mempool * mp =
2540                                                 mbuf_pool_find
2541                                                         (rxring_numa[pi], 0);
2542                                         if (mp == NULL) {
2543                                                 fprintf(stderr,
2544                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2545                                                         rxring_numa[pi]);
2546                                                 return -1;
2547                                         }
2548
2549                                         diag = rx_queue_setup(pi, qi,
2550                                              port->nb_rx_desc[qi],
2551                                              rxring_numa[pi],
2552                                              &(port->rx_conf[qi]),
2553                                              mp);
2554                                 } else {
2555                                         struct rte_mempool *mp =
2556                                                 mbuf_pool_find
2557                                                         (port->socket_id, 0);
2558                                         if (mp == NULL) {
2559                                                 fprintf(stderr,
2560                                                         "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2561                                                         port->socket_id);
2562                                                 return -1;
2563                                         }
2564                                         diag = rx_queue_setup(pi, qi,
2565                                              port->nb_rx_desc[qi],
2566                                              port->socket_id,
2567                                              &(port->rx_conf[qi]),
2568                                              mp);
2569                                 }
2570                                 if (diag == 0)
2571                                         continue;
2572
2573                                 /* Fail to setup rx queue, return */
2574                                 if (rte_atomic16_cmpset(&(port->port_status),
2575                                                         RTE_PORT_HANDLING,
2576                                                         RTE_PORT_STOPPED) == 0)
2577                                         fprintf(stderr,
2578                                                 "Port %d can not be set back to stopped\n",
2579                                                 pi);
2580                                 fprintf(stderr,
2581                                         "Fail to configure port %d rx queues\n",
2582                                         pi);
2583                                 /* try to reconfigure queues next time */
2584                                 port->need_reconfig_queues = 1;
2585                                 return -1;
2586                         }
2587                         /* setup hairpin queues */
2588                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2589                                 return -1;
2590                 }
2591                 configure_rxtx_dump_callbacks(verbose_level);
2592                 if (clear_ptypes) {
2593                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2594                                         NULL, 0);
2595                         if (diag < 0)
2596                                 fprintf(stderr,
2597                                         "Port %d: Failed to disable Ptype parsing\n",
2598                                         pi);
2599                 }
2600
2601                 p_pi = pi;
2602                 cnt_pi++;
2603
2604                 /* start port */
2605                 diag = rte_eth_dev_start(pi);
2606                 if (diag < 0) {
2607                         fprintf(stderr, "Fail to start port %d: %s\n",
2608                                 pi, rte_strerror(-diag));
2609
2610                         /* Fail to setup rx queue, return */
2611                         if (rte_atomic16_cmpset(&(port->port_status),
2612                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2613                                 fprintf(stderr,
2614                                         "Port %d can not be set back to stopped\n",
2615                                         pi);
2616                         continue;
2617                 }
2618
2619                 if (rte_atomic16_cmpset(&(port->port_status),
2620                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2621                         fprintf(stderr, "Port %d can not be set into started\n",
2622                                 pi);
2623
2624                 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
2625                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2626                                 port->eth_addr.addr_bytes[0],
2627                                 port->eth_addr.addr_bytes[1],
2628                                 port->eth_addr.addr_bytes[2],
2629                                 port->eth_addr.addr_bytes[3],
2630                                 port->eth_addr.addr_bytes[4],
2631                                 port->eth_addr.addr_bytes[5]);
2632
2633                 /* at least one port started, need checking link status */
2634                 need_check_link_status = 1;
2635
2636                 pl[cfg_pi++] = pi;
2637         }
2638
2639         if (need_check_link_status == 1 && !no_link_check)
2640                 check_all_ports_link_status(RTE_PORT_ALL);
2641         else if (need_check_link_status == 0)
2642                 fprintf(stderr, "Please stop the ports first\n");
2643
2644         if (hairpin_mode & 0xf) {
2645                 uint16_t i;
2646                 int j;
2647
2648                 /* bind all started hairpin ports */
2649                 for (i = 0; i < cfg_pi; i++) {
2650                         pi = pl[i];
2651                         /* bind current Tx to all peer Rx */
2652                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2653                                                         RTE_MAX_ETHPORTS, 1);
2654                         if (peer_pi < 0)
2655                                 return peer_pi;
2656                         for (j = 0; j < peer_pi; j++) {
2657                                 if (!port_is_started(peer_pl[j]))
2658                                         continue;
2659                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2660                                 if (diag < 0) {
2661                                         fprintf(stderr,
2662                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
2663                                                 pi, peer_pl[j],
2664                                                 rte_strerror(-diag));
2665                                         return -1;
2666                                 }
2667                         }
2668                         /* bind all peer Tx to current Rx */
2669                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2670                                                         RTE_MAX_ETHPORTS, 0);
2671                         if (peer_pi < 0)
2672                                 return peer_pi;
2673                         for (j = 0; j < peer_pi; j++) {
2674                                 if (!port_is_started(peer_pl[j]))
2675                                         continue;
2676                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2677                                 if (diag < 0) {
2678                                         fprintf(stderr,
2679                                                 "Error during binding hairpin Tx port %u to %u: %s\n",
2680                                                 peer_pl[j], pi,
2681                                                 rte_strerror(-diag));
2682                                         return -1;
2683                                 }
2684                         }
2685                 }
2686         }
2687
2688         printf("Done\n");
2689         return 0;
2690 }
2691
2692 void
2693 stop_port(portid_t pid)
2694 {
2695         portid_t pi;
2696         struct rte_port *port;
2697         int need_check_link_status = 0;
2698         portid_t peer_pl[RTE_MAX_ETHPORTS];
2699         int peer_pi;
2700
2701         if (port_id_is_invalid(pid, ENABLED_WARN))
2702                 return;
2703
2704         printf("Stopping ports...\n");
2705
2706         RTE_ETH_FOREACH_DEV(pi) {
2707                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2708                         continue;
2709
2710                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2711                         fprintf(stderr,
2712                                 "Please remove port %d from forwarding configuration.\n",
2713                                 pi);
2714                         continue;
2715                 }
2716
2717                 if (port_is_bonding_slave(pi)) {
2718                         fprintf(stderr,
2719                                 "Please remove port %d from bonded device.\n",
2720                                 pi);
2721                         continue;
2722                 }
2723
2724                 port = &ports[pi];
2725                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2726                                                 RTE_PORT_HANDLING) == 0)
2727                         continue;
2728
2729                 if (hairpin_mode & 0xf) {
2730                         int j;
2731
2732                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2733                         /* unbind all peer Tx from current Rx */
2734                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2735                                                         RTE_MAX_ETHPORTS, 0);
2736                         if (peer_pi < 0)
2737                                 continue;
2738                         for (j = 0; j < peer_pi; j++) {
2739                                 if (!port_is_started(peer_pl[j]))
2740                                         continue;
2741                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2742                         }
2743                 }
2744
2745                 if (port->flow_list)
2746                         port_flow_flush(pi);
2747
2748                 if (rte_eth_dev_stop(pi) != 0)
2749                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2750                                 pi);
2751
2752                 if (rte_atomic16_cmpset(&(port->port_status),
2753                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2754                         fprintf(stderr, "Port %d can not be set into stopped\n",
2755                                 pi);
2756                 need_check_link_status = 1;
2757         }
2758         if (need_check_link_status && !no_link_check)
2759                 check_all_ports_link_status(RTE_PORT_ALL);
2760
2761         printf("Done\n");
2762 }
2763
2764 static void
2765 remove_invalid_ports_in(portid_t *array, portid_t *total)
2766 {
2767         portid_t i;
2768         portid_t new_total = 0;
2769
2770         for (i = 0; i < *total; i++)
2771                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2772                         array[new_total] = array[i];
2773                         new_total++;
2774                 }
2775         *total = new_total;
2776 }
2777
2778 static void
2779 remove_invalid_ports(void)
2780 {
2781         remove_invalid_ports_in(ports_ids, &nb_ports);
2782         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2783         nb_cfg_ports = nb_fwd_ports;
2784 }
2785
2786 void
2787 close_port(portid_t pid)
2788 {
2789         portid_t pi;
2790         struct rte_port *port;
2791
2792         if (port_id_is_invalid(pid, ENABLED_WARN))
2793                 return;
2794
2795         printf("Closing ports...\n");
2796
2797         RTE_ETH_FOREACH_DEV(pi) {
2798                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2799                         continue;
2800
2801                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2802                         fprintf(stderr,
2803                                 "Please remove port %d from forwarding configuration.\n",
2804                                 pi);
2805                         continue;
2806                 }
2807
2808                 if (port_is_bonding_slave(pi)) {
2809                         fprintf(stderr,
2810                                 "Please remove port %d from bonded device.\n",
2811                                 pi);
2812                         continue;
2813                 }
2814
2815                 port = &ports[pi];
2816                 if (rte_atomic16_cmpset(&(port->port_status),
2817                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2818                         fprintf(stderr, "Port %d is already closed\n", pi);
2819                         continue;
2820                 }
2821
2822                 port_flow_flush(pi);
2823                 rte_eth_dev_close(pi);
2824         }
2825
2826         remove_invalid_ports();
2827         printf("Done\n");
2828 }
2829
2830 void
2831 reset_port(portid_t pid)
2832 {
2833         int diag;
2834         portid_t pi;
2835         struct rte_port *port;
2836
2837         if (port_id_is_invalid(pid, ENABLED_WARN))
2838                 return;
2839
2840         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2841                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2842                 fprintf(stderr,
2843                         "Can not reset port(s), please stop port(s) first.\n");
2844                 return;
2845         }
2846
2847         printf("Resetting ports...\n");
2848
2849         RTE_ETH_FOREACH_DEV(pi) {
2850                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2851                         continue;
2852
2853                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2854                         fprintf(stderr,
2855                                 "Please remove port %d from forwarding configuration.\n",
2856                                 pi);
2857                         continue;
2858                 }
2859
2860                 if (port_is_bonding_slave(pi)) {
2861                         fprintf(stderr,
2862                                 "Please remove port %d from bonded device.\n",
2863                                 pi);
2864                         continue;
2865                 }
2866
2867                 diag = rte_eth_dev_reset(pi);
2868                 if (diag == 0) {
2869                         port = &ports[pi];
2870                         port->need_reconfig = 1;
2871                         port->need_reconfig_queues = 1;
2872                 } else {
2873                         fprintf(stderr, "Failed to reset port %d. diag=%d\n",
2874                                 pi, diag);
2875                 }
2876         }
2877
2878         printf("Done\n");
2879 }
2880
2881 void
2882 attach_port(char *identifier)
2883 {
2884         portid_t pi;
2885         struct rte_dev_iterator iterator;
2886
2887         printf("Attaching a new port...\n");
2888
2889         if (identifier == NULL) {
2890                 fprintf(stderr, "Invalid parameters are specified\n");
2891                 return;
2892         }
2893
2894         if (rte_dev_probe(identifier) < 0) {
2895                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2896                 return;
2897         }
2898
2899         /* first attach mode: event */
2900         if (setup_on_probe_event) {
2901                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2902                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2903                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2904                                         ports[pi].need_setup != 0)
2905                                 setup_attached_port(pi);
2906                 return;
2907         }
2908
2909         /* second attach mode: iterator */
2910         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2911                 /* setup ports matching the devargs used for probing */
2912                 if (port_is_forwarding(pi))
2913                         continue; /* port was already attached before */
2914                 setup_attached_port(pi);
2915         }
2916 }
2917
2918 static void
2919 setup_attached_port(portid_t pi)
2920 {
2921         unsigned int socket_id;
2922         int ret;
2923
2924         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2925         /* if socket_id is invalid, set to the first available socket. */
2926         if (check_socket_id(socket_id) < 0)
2927                 socket_id = socket_ids[0];
2928         reconfig(pi, socket_id);
2929         ret = rte_eth_promiscuous_enable(pi);
2930         if (ret != 0)
2931                 fprintf(stderr,
2932                         "Error during enabling promiscuous mode for port %u: %s - ignore\n",
2933                         pi, rte_strerror(-ret));
2934
2935         ports_ids[nb_ports++] = pi;
2936         fwd_ports_ids[nb_fwd_ports++] = pi;
2937         nb_cfg_ports = nb_fwd_ports;
2938         ports[pi].need_setup = 0;
2939         ports[pi].port_status = RTE_PORT_STOPPED;
2940
2941         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2942         printf("Done\n");
2943 }
2944
2945 static void
2946 detach_device(struct rte_device *dev)
2947 {
2948         portid_t sibling;
2949
2950         if (dev == NULL) {
2951                 fprintf(stderr, "Device already removed\n");
2952                 return;
2953         }
2954
2955         printf("Removing a device...\n");
2956
2957         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2958                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2959                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2960                                 fprintf(stderr, "Port %u not stopped\n",
2961                                         sibling);
2962                                 return;
2963                         }
2964                         port_flow_flush(sibling);
2965                 }
2966         }
2967
2968         if (rte_dev_remove(dev) < 0) {
2969                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2970                 return;
2971         }
2972         remove_invalid_ports();
2973
2974         printf("Device is detached\n");
2975         printf("Now total ports is %d\n", nb_ports);
2976         printf("Done\n");
2977         return;
2978 }
2979
2980 void
2981 detach_port_device(portid_t port_id)
2982 {
2983         int ret;
2984         struct rte_eth_dev_info dev_info;
2985
2986         if (port_id_is_invalid(port_id, ENABLED_WARN))
2987                 return;
2988
2989         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2990                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2991                         fprintf(stderr, "Port not stopped\n");
2992                         return;
2993                 }
2994                 fprintf(stderr, "Port was not closed\n");
2995         }
2996
2997         ret = eth_dev_info_get_print_err(port_id, &dev_info);
2998         if (ret != 0) {
2999                 TESTPMD_LOG(ERR,
3000                         "Failed to get device info for port %d, not detaching\n",
3001                         port_id);
3002                 return;
3003         }
3004         detach_device(dev_info.device);
3005 }
3006
3007 void
3008 detach_devargs(char *identifier)
3009 {
3010         struct rte_dev_iterator iterator;
3011         struct rte_devargs da;
3012         portid_t port_id;
3013
3014         printf("Removing a device...\n");
3015
3016         memset(&da, 0, sizeof(da));
3017         if (rte_devargs_parsef(&da, "%s", identifier)) {
3018                 fprintf(stderr, "cannot parse identifier\n");
3019                 return;
3020         }
3021
3022         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3023                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3024                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3025                                 fprintf(stderr, "Port %u not stopped\n",
3026                                         port_id);
3027                                 rte_eth_iterator_cleanup(&iterator);
3028                                 rte_devargs_reset(&da);
3029                                 return;
3030                         }
3031                         port_flow_flush(port_id);
3032                 }
3033         }
3034
3035         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3036                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3037                             da.name, da.bus->name);
3038                 rte_devargs_reset(&da);
3039                 return;
3040         }
3041
3042         remove_invalid_ports();
3043
3044         printf("Device %s is detached\n", identifier);
3045         printf("Now total ports is %d\n", nb_ports);
3046         printf("Done\n");
3047         rte_devargs_reset(&da);
3048 }
3049
3050 void
3051 pmd_test_exit(void)
3052 {
3053         portid_t pt_id;
3054         unsigned int i;
3055         int ret;
3056
3057         if (test_done == 0)
3058                 stop_packet_forwarding();
3059
3060 #ifndef RTE_EXEC_ENV_WINDOWS
3061         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3062                 if (mempools[i]) {
3063                         if (mp_alloc_type == MP_ALLOC_ANON)
3064                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3065                                                      NULL);
3066                 }
3067         }
3068 #endif
3069         if (ports != NULL) {
3070                 no_link_check = 1;
3071                 RTE_ETH_FOREACH_DEV(pt_id) {
3072                         printf("\nStopping port %d...\n", pt_id);
3073                         fflush(stdout);
3074                         stop_port(pt_id);
3075                 }
3076                 RTE_ETH_FOREACH_DEV(pt_id) {
3077                         printf("\nShutting down port %d...\n", pt_id);
3078                         fflush(stdout);
3079                         close_port(pt_id);
3080                 }
3081         }
3082
3083         if (hot_plug) {
3084                 ret = rte_dev_event_monitor_stop();
3085                 if (ret) {
3086                         RTE_LOG(ERR, EAL,
3087                                 "fail to stop device event monitor.");
3088                         return;
3089                 }
3090
3091                 ret = rte_dev_event_callback_unregister(NULL,
3092                         dev_event_callback, NULL);
3093                 if (ret < 0) {
3094                         RTE_LOG(ERR, EAL,
3095                                 "fail to unregister device event callback.\n");
3096                         return;
3097                 }
3098
3099                 ret = rte_dev_hotplug_handle_disable();
3100                 if (ret) {
3101                         RTE_LOG(ERR, EAL,
3102                                 "fail to disable hotplug handling.\n");
3103                         return;
3104                 }
3105         }
3106         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3107                 if (mempools[i])
3108                         rte_mempool_free(mempools[i]);
3109         }
3110
3111         printf("\nBye...\n");
3112 }
3113
3114 typedef void (*cmd_func_t)(void);
3115 struct pmd_test_command {
3116         const char *cmd_name;
3117         cmd_func_t cmd_func;
3118 };
3119
3120 /* Check the link status of all ports in up to 9s, and print them finally */
3121 static void
3122 check_all_ports_link_status(uint32_t port_mask)
3123 {
3124 #define CHECK_INTERVAL 100 /* 100ms */
3125 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3126         portid_t portid;
3127         uint8_t count, all_ports_up, print_flag = 0;
3128         struct rte_eth_link link;
3129         int ret;
3130         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3131
3132         printf("Checking link statuses...\n");
3133         fflush(stdout);
3134         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3135                 all_ports_up = 1;
3136                 RTE_ETH_FOREACH_DEV(portid) {
3137                         if ((port_mask & (1 << portid)) == 0)
3138                                 continue;
3139                         memset(&link, 0, sizeof(link));
3140                         ret = rte_eth_link_get_nowait(portid, &link);
3141                         if (ret < 0) {
3142                                 all_ports_up = 0;
3143                                 if (print_flag == 1)
3144                                         fprintf(stderr,
3145                                                 "Port %u link get failed: %s\n",
3146                                                 portid, rte_strerror(-ret));
3147                                 continue;
3148                         }
3149                         /* print link status if flag set */
3150                         if (print_flag == 1) {
3151                                 rte_eth_link_to_str(link_status,
3152                                         sizeof(link_status), &link);
3153                                 printf("Port %d %s\n", portid, link_status);
3154                                 continue;
3155                         }
3156                         /* clear all_ports_up flag if any link down */
3157                         if (link.link_status == ETH_LINK_DOWN) {
3158                                 all_ports_up = 0;
3159                                 break;
3160                         }
3161                 }
3162                 /* after finally printing all link status, get out */
3163                 if (print_flag == 1)
3164                         break;
3165
3166                 if (all_ports_up == 0) {
3167                         fflush(stdout);
3168                         rte_delay_ms(CHECK_INTERVAL);
3169                 }
3170
3171                 /* set the print_flag if all ports up or timeout */
3172                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3173                         print_flag = 1;
3174                 }
3175
3176                 if (lsc_interrupt)
3177                         break;
3178         }
3179 }
3180
3181 static void
3182 rmv_port_callback(void *arg)
3183 {
3184         int need_to_start = 0;
3185         int org_no_link_check = no_link_check;
3186         portid_t port_id = (intptr_t)arg;
3187         struct rte_eth_dev_info dev_info;
3188         int ret;
3189
3190         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3191
3192         if (!test_done && port_is_forwarding(port_id)) {
3193                 need_to_start = 1;
3194                 stop_packet_forwarding();
3195         }
3196         no_link_check = 1;
3197         stop_port(port_id);
3198         no_link_check = org_no_link_check;
3199
3200         ret = eth_dev_info_get_print_err(port_id, &dev_info);
3201         if (ret != 0)
3202                 TESTPMD_LOG(ERR,
3203                         "Failed to get device info for port %d, not detaching\n",
3204                         port_id);
3205         else {
3206                 struct rte_device *device = dev_info.device;
3207                 close_port(port_id);
3208                 detach_device(device); /* might be already removed or have more ports */
3209         }
3210         if (need_to_start)
3211                 start_packet_forwarding(0);
3212 }
3213
3214 /* This function is used by the interrupt thread */
3215 static int
3216 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3217                   void *ret_param)
3218 {
3219         RTE_SET_USED(param);
3220         RTE_SET_USED(ret_param);
3221
3222         if (type >= RTE_ETH_EVENT_MAX) {
3223                 fprintf(stderr,
3224                         "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3225                         port_id, __func__, type);
3226                 fflush(stderr);
3227         } else if (event_print_mask & (UINT32_C(1) << type)) {
3228                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3229                         eth_event_desc[type]);
3230                 fflush(stdout);
3231         }
3232
3233         switch (type) {
3234         case RTE_ETH_EVENT_NEW:
3235                 ports[port_id].need_setup = 1;
3236                 ports[port_id].port_status = RTE_PORT_HANDLING;
3237                 break;
3238         case RTE_ETH_EVENT_INTR_RMV:
3239                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3240                         break;
3241                 if (rte_eal_alarm_set(100000,
3242                                 rmv_port_callback, (void *)(intptr_t)port_id))
3243                         fprintf(stderr,
3244                                 "Could not set up deferred device removal\n");
3245                 break;
3246         case RTE_ETH_EVENT_DESTROY:
3247                 ports[port_id].port_status = RTE_PORT_CLOSED;
3248                 printf("Port %u is closed\n", port_id);
3249                 break;
3250         default:
3251                 break;
3252         }
3253         return 0;
3254 }
3255
3256 static int
3257 register_eth_event_callback(void)
3258 {
3259         int ret;
3260         enum rte_eth_event_type event;
3261
3262         for (event = RTE_ETH_EVENT_UNKNOWN;
3263                         event < RTE_ETH_EVENT_MAX; event++) {
3264                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3265                                 event,
3266                                 eth_event_callback,
3267                                 NULL);
3268                 if (ret != 0) {
3269                         TESTPMD_LOG(ERR, "Failed to register callback for "
3270                                         "%s event\n", eth_event_desc[event]);
3271                         return -1;
3272                 }
3273         }
3274
3275         return 0;
3276 }
3277
3278 /* This function is used by the interrupt thread */
3279 static void
3280 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3281                              __rte_unused void *arg)
3282 {
3283         uint16_t port_id;
3284         int ret;
3285
3286         if (type >= RTE_DEV_EVENT_MAX) {
3287                 fprintf(stderr, "%s called upon invalid event %d\n",
3288                         __func__, type);
3289                 fflush(stderr);
3290         }
3291
3292         switch (type) {
3293         case RTE_DEV_EVENT_REMOVE:
3294                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3295                         device_name);
3296                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3297                 if (ret) {
3298                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3299                                 device_name);
3300                         return;
3301                 }
3302                 /*
3303                  * Because the user's callback is invoked in eal interrupt
3304                  * callback, the interrupt callback need to be finished before
3305                  * it can be unregistered when detaching device. So finish
3306                  * callback soon and use a deferred removal to detach device
3307                  * is need. It is a workaround, once the device detaching be
3308                  * moved into the eal in the future, the deferred removal could
3309                  * be deleted.
3310                  */
3311                 if (rte_eal_alarm_set(100000,
3312                                 rmv_port_callback, (void *)(intptr_t)port_id))
3313                         RTE_LOG(ERR, EAL,
3314                                 "Could not set up deferred device removal\n");
3315                 break;
3316         case RTE_DEV_EVENT_ADD:
3317                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3318                         device_name);
3319                 /* TODO: After finish kernel driver binding,
3320                  * begin to attach port.
3321                  */
3322                 break;
3323         default:
3324                 break;
3325         }
3326 }
3327
3328 static void
3329 rxtx_port_config(struct rte_port *port)
3330 {
3331         uint16_t qid;
3332         uint64_t offloads;
3333
3334         for (qid = 0; qid < nb_rxq; qid++) {
3335                 offloads = port->rx_conf[qid].offloads;
3336                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3337                 if (offloads != 0)
3338                         port->rx_conf[qid].offloads = offloads;
3339
3340                 /* Check if any Rx parameters have been passed */
3341                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3342                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3343
3344                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3345                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3346
3347                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3348                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3349
3350                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3351                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3352
3353                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3354                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3355
3356                 port->nb_rx_desc[qid] = nb_rxd;
3357         }
3358
3359         for (qid = 0; qid < nb_txq; qid++) {
3360                 offloads = port->tx_conf[qid].offloads;
3361                 port->tx_conf[qid] = port->dev_info.default_txconf;
3362                 if (offloads != 0)
3363                         port->tx_conf[qid].offloads = offloads;
3364
3365                 /* Check if any Tx parameters have been passed */
3366                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3367                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3368
3369                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3370                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3371
3372                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3373                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3374
3375                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3376                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3377
3378                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3379                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3380
3381                 port->nb_tx_desc[qid] = nb_txd;
3382         }
3383 }
3384
3385 /*
3386  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3387  * MTU is also aligned if JUMBO_FRAME offload is not set.
3388  *
3389  * port->dev_info should be set before calling this function.
3390  *
3391  * return 0 on success, negative on error
3392  */
3393 int
3394 update_jumbo_frame_offload(portid_t portid)
3395 {
3396         struct rte_port *port = &ports[portid];
3397         uint32_t eth_overhead;
3398         uint64_t rx_offloads;
3399         int ret;
3400         bool on;
3401
3402         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3403         if (port->dev_info.max_mtu != UINT16_MAX &&
3404             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3405                 eth_overhead = port->dev_info.max_rx_pktlen -
3406                                 port->dev_info.max_mtu;
3407         else
3408                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3409
3410         rx_offloads = port->dev_conf.rxmode.offloads;
3411
3412         /* Default config value is 0 to use PMD specific overhead */
3413         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3414                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3415
3416         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3417                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3418                 on = false;
3419         } else {
3420                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3421                         fprintf(stderr,
3422                                 "Frame size (%u) is not supported by port %u\n",
3423                                 port->dev_conf.rxmode.max_rx_pkt_len,
3424                                 portid);
3425                         return -1;
3426                 }
3427                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3428                 on = true;
3429         }
3430
3431         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3432                 uint16_t qid;
3433
3434                 port->dev_conf.rxmode.offloads = rx_offloads;
3435
3436                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3437                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3438                         if (on)
3439                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3440                         else
3441                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3442                 }
3443         }
3444
3445         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3446          * if unset do it here
3447          */
3448         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3449                 ret = rte_eth_dev_set_mtu(portid,
3450                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3451                 if (ret)
3452                         fprintf(stderr,
3453                                 "Failed to set MTU to %u for port %u\n",
3454                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3455                                 portid);
3456         }
3457
3458         return 0;
3459 }
3460
3461 void
3462 init_port_config(void)
3463 {
3464         portid_t pid;
3465         struct rte_port *port;
3466         int ret;
3467
3468         RTE_ETH_FOREACH_DEV(pid) {
3469                 port = &ports[pid];
3470                 port->dev_conf.fdir_conf = fdir_conf;
3471
3472                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3473                 if (ret != 0)
3474                         return;
3475
3476                 if (nb_rxq > 1) {
3477                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3478                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3479                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3480                 } else {
3481                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3482                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3483                 }
3484
3485                 if (port->dcb_flag == 0) {
3486                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3487                                 port->dev_conf.rxmode.mq_mode =
3488                                         (enum rte_eth_rx_mq_mode)
3489                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3490                         else
3491                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3492                 }
3493
3494                 rxtx_port_config(port);
3495
3496                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3497                 if (ret != 0)
3498                         return;
3499
3500 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3501                 rte_pmd_ixgbe_bypass_init(pid);
3502 #endif
3503
3504                 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3505                         port->dev_conf.intr_conf.lsc = 1;
3506                 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3507                         port->dev_conf.intr_conf.rmv = 1;
3508         }
3509 }
3510
3511 void set_port_slave_flag(portid_t slave_pid)
3512 {
3513         struct rte_port *port;
3514
3515         port = &ports[slave_pid];
3516         port->slave_flag = 1;
3517 }
3518
3519 void clear_port_slave_flag(portid_t slave_pid)
3520 {
3521         struct rte_port *port;
3522
3523         port = &ports[slave_pid];
3524         port->slave_flag = 0;
3525 }
3526
3527 uint8_t port_is_bonding_slave(portid_t slave_pid)
3528 {
3529         struct rte_port *port;
3530         struct rte_eth_dev_info dev_info;
3531         int ret;
3532
3533         port = &ports[slave_pid];
3534         ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3535         if (ret != 0) {
3536                 TESTPMD_LOG(ERR,
3537                         "Failed to get device info for port id %d,"
3538                         "cannot determine if the port is a bonded slave",
3539                         slave_pid);
3540                 return 0;
3541         }
3542         if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3543                 return 1;
3544         return 0;
3545 }
3546
3547 const uint16_t vlan_tags[] = {
3548                 0,  1,  2,  3,  4,  5,  6,  7,
3549                 8,  9, 10, 11,  12, 13, 14, 15,
3550                 16, 17, 18, 19, 20, 21, 22, 23,
3551                 24, 25, 26, 27, 28, 29, 30, 31
3552 };
3553
3554 static  int
3555 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3556                  enum dcb_mode_enable dcb_mode,
3557                  enum rte_eth_nb_tcs num_tcs,
3558                  uint8_t pfc_en)
3559 {
3560         uint8_t i;
3561         int32_t rc;
3562         struct rte_eth_rss_conf rss_conf;
3563
3564         /*
3565          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3566          * given above, and the number of traffic classes available for use.
3567          */
3568         if (dcb_mode == DCB_VT_ENABLED) {
3569                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3570                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3571                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3572                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3573
3574                 /* VMDQ+DCB RX and TX configurations */
3575                 vmdq_rx_conf->enable_default_pool = 0;
3576                 vmdq_rx_conf->default_pool = 0;
3577                 vmdq_rx_conf->nb_queue_pools =
3578                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3579                 vmdq_tx_conf->nb_queue_pools =
3580                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3581
3582                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3583                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3584                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3585                         vmdq_rx_conf->pool_map[i].pools =
3586                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3587                 }
3588                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3589                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3590                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3591                 }
3592
3593                 /* set DCB mode of RX and TX of multiple queues */
3594                 eth_conf->rxmode.mq_mode =
3595                                 (enum rte_eth_rx_mq_mode)
3596                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3597                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3598         } else {
3599                 struct rte_eth_dcb_rx_conf *rx_conf =
3600                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3601                 struct rte_eth_dcb_tx_conf *tx_conf =
3602                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3603
3604                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3605
3606                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3607                 if (rc != 0)
3608                         return rc;
3609
3610                 rx_conf->nb_tcs = num_tcs;
3611                 tx_conf->nb_tcs = num_tcs;
3612
3613                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3614                         rx_conf->dcb_tc[i] = i % num_tcs;
3615                         tx_conf->dcb_tc[i] = i % num_tcs;
3616                 }
3617
3618                 eth_conf->rxmode.mq_mode =
3619                                 (enum rte_eth_rx_mq_mode)
3620                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3621                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3622                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3623         }
3624
3625         if (pfc_en)
3626                 eth_conf->dcb_capability_en =
3627                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3628         else
3629                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3630
3631         return 0;
3632 }
3633
3634 int
3635 init_port_dcb_config(portid_t pid,
3636                      enum dcb_mode_enable dcb_mode,
3637                      enum rte_eth_nb_tcs num_tcs,
3638                      uint8_t pfc_en)
3639 {
3640         struct rte_eth_conf port_conf;
3641         struct rte_port *rte_port;
3642         int retval;
3643         uint16_t i;
3644
3645         rte_port = &ports[pid];
3646
3647         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3648
3649         port_conf.rxmode = rte_port->dev_conf.rxmode;
3650         port_conf.txmode = rte_port->dev_conf.txmode;
3651
3652         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3653         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3654         if (retval < 0)
3655                 return retval;
3656         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3657
3658         /* re-configure the device . */
3659         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3660         if (retval < 0)
3661                 return retval;
3662
3663         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3664         if (retval != 0)
3665                 return retval;
3666
3667         /* If dev_info.vmdq_pool_base is greater than 0,
3668          * the queue id of vmdq pools is started after pf queues.
3669          */
3670         if (dcb_mode == DCB_VT_ENABLED &&
3671             rte_port->dev_info.vmdq_pool_base > 0) {
3672                 fprintf(stderr,
3673                         "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
3674                         pid);
3675                 return -1;
3676         }
3677
3678         /* Assume the ports in testpmd have the same dcb capability
3679          * and has the same number of rxq and txq in dcb mode
3680          */
3681         if (dcb_mode == DCB_VT_ENABLED) {
3682                 if (rte_port->dev_info.max_vfs > 0) {
3683                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3684                         nb_txq = rte_port->dev_info.nb_tx_queues;
3685                 } else {
3686                         nb_rxq = rte_port->dev_info.max_rx_queues;
3687                         nb_txq = rte_port->dev_info.max_tx_queues;
3688                 }
3689         } else {
3690                 /*if vt is disabled, use all pf queues */
3691                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3692                         nb_rxq = rte_port->dev_info.max_rx_queues;
3693                         nb_txq = rte_port->dev_info.max_tx_queues;
3694                 } else {
3695                         nb_rxq = (queueid_t)num_tcs;
3696                         nb_txq = (queueid_t)num_tcs;
3697
3698                 }
3699         }
3700         rx_free_thresh = 64;
3701
3702         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3703
3704         rxtx_port_config(rte_port);
3705         /* VLAN filter */
3706         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3707         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3708                 rx_vft_set(pid, vlan_tags[i], 1);
3709
3710         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3711         if (retval != 0)
3712                 return retval;
3713
3714         rte_port->dcb_flag = 1;
3715
3716         /* Enter DCB configuration status */
3717         dcb_config = 1;
3718
3719         return 0;
3720 }
3721
3722 static void
3723 init_port(void)
3724 {
3725         int i;
3726
3727         /* Configuration of Ethernet ports. */
3728         ports = rte_zmalloc("testpmd: ports",
3729                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3730                             RTE_CACHE_LINE_SIZE);
3731         if (ports == NULL) {
3732                 rte_exit(EXIT_FAILURE,
3733                                 "rte_zmalloc(%d struct rte_port) failed\n",
3734                                 RTE_MAX_ETHPORTS);
3735         }
3736         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3737                 LIST_INIT(&ports[i].flow_tunnel_list);
3738         /* Initialize ports NUMA structures */
3739         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3740         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3741         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3742 }
3743
3744 static void
3745 force_quit(void)
3746 {
3747         pmd_test_exit();
3748         prompt_exit();
3749 }
3750
3751 static void
3752 print_stats(void)
3753 {
3754         uint8_t i;
3755         const char clr[] = { 27, '[', '2', 'J', '\0' };
3756         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3757
3758         /* Clear screen and move to top left */
3759         printf("%s%s", clr, top_left);
3760
3761         printf("\nPort statistics ====================================");
3762         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3763                 nic_stats_display(fwd_ports_ids[i]);
3764
3765         fflush(stdout);
3766 }
3767
3768 static void
3769 signal_handler(int signum)
3770 {
3771         if (signum == SIGINT || signum == SIGTERM) {
3772                 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
3773                         signum);
3774 #ifdef RTE_LIB_PDUMP
3775                 /* uninitialize packet capture framework */
3776                 rte_pdump_uninit();
3777 #endif
3778 #ifdef RTE_LIB_LATENCYSTATS
3779                 if (latencystats_enabled != 0)
3780                         rte_latencystats_uninit();
3781 #endif
3782                 force_quit();
3783                 /* Set flag to indicate the force termination. */
3784                 f_quit = 1;
3785                 /* exit with the expected status */
3786 #ifndef RTE_EXEC_ENV_WINDOWS
3787                 signal(signum, SIG_DFL);
3788                 kill(getpid(), signum);
3789 #endif
3790         }
3791 }
3792
3793 int
3794 main(int argc, char** argv)
3795 {
3796         int diag;
3797         portid_t port_id;
3798         uint16_t count;
3799         int ret;
3800
3801         signal(SIGINT, signal_handler);
3802         signal(SIGTERM, signal_handler);
3803
3804         testpmd_logtype = rte_log_register("testpmd");
3805         if (testpmd_logtype < 0)
3806                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3807         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3808
3809         diag = rte_eal_init(argc, argv);
3810         if (diag < 0)
3811                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3812                          rte_strerror(rte_errno));
3813
3814         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3815                 rte_exit(EXIT_FAILURE,
3816                          "Secondary process type not supported.\n");
3817
3818         ret = register_eth_event_callback();
3819         if (ret != 0)
3820                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3821
3822 #ifdef RTE_LIB_PDUMP
3823         /* initialize packet capture framework */
3824         rte_pdump_init();
3825 #endif
3826
3827         count = 0;
3828         RTE_ETH_FOREACH_DEV(port_id) {
3829                 ports_ids[count] = port_id;
3830                 count++;
3831         }
3832         nb_ports = (portid_t) count;
3833         if (nb_ports == 0)
3834                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3835
3836         /* allocate port structures, and init them */
3837         init_port();
3838
3839         set_def_fwd_config();
3840         if (nb_lcores == 0)
3841                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3842                          "Check the core mask argument\n");
3843
3844         /* Bitrate/latency stats disabled by default */
3845 #ifdef RTE_LIB_BITRATESTATS
3846         bitrate_enabled = 0;
3847 #endif
3848 #ifdef RTE_LIB_LATENCYSTATS
3849         latencystats_enabled = 0;
3850 #endif
3851
3852         /* on FreeBSD, mlockall() is disabled by default */
3853 #ifdef RTE_EXEC_ENV_FREEBSD
3854         do_mlockall = 0;
3855 #else
3856         do_mlockall = 1;
3857 #endif
3858
3859         argc -= diag;
3860         argv += diag;
3861         if (argc > 1)
3862                 launch_args_parse(argc, argv);
3863
3864 #ifndef RTE_EXEC_ENV_WINDOWS
3865         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3866                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3867                         strerror(errno));
3868         }
3869 #endif
3870
3871         if (tx_first && interactive)
3872                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3873                                 "interactive mode.\n");
3874
3875         if (tx_first && lsc_interrupt) {
3876                 fprintf(stderr,
3877                         "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
3878                 lsc_interrupt = 0;
3879         }
3880
3881         if (!nb_rxq && !nb_txq)
3882                 fprintf(stderr,
3883                         "Warning: Either rx or tx queues should be non-zero\n");
3884
3885         if (nb_rxq > 1 && nb_rxq > nb_txq)
3886                 fprintf(stderr,
3887                         "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
3888                         nb_rxq, nb_txq);
3889
3890         init_config();
3891
3892         if (hot_plug) {
3893                 ret = rte_dev_hotplug_handle_enable();
3894                 if (ret) {
3895                         RTE_LOG(ERR, EAL,
3896                                 "fail to enable hotplug handling.");
3897                         return -1;
3898                 }
3899
3900                 ret = rte_dev_event_monitor_start();
3901                 if (ret) {
3902                         RTE_LOG(ERR, EAL,
3903                                 "fail to start device event monitoring.");
3904                         return -1;
3905                 }
3906
3907                 ret = rte_dev_event_callback_register(NULL,
3908                         dev_event_callback, NULL);
3909                 if (ret) {
3910                         RTE_LOG(ERR, EAL,
3911                                 "fail  to register device event callback\n");
3912                         return -1;
3913                 }
3914         }
3915
3916         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3917                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3918
3919         /* set all ports to promiscuous mode by default */
3920         RTE_ETH_FOREACH_DEV(port_id) {
3921                 ret = rte_eth_promiscuous_enable(port_id);
3922                 if (ret != 0)
3923                         fprintf(stderr,
3924                                 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3925                                 port_id, rte_strerror(-ret));
3926         }
3927
3928         /* Init metrics library */
3929         rte_metrics_init(rte_socket_id());
3930
3931 #ifdef RTE_LIB_LATENCYSTATS
3932         if (latencystats_enabled != 0) {
3933                 int ret = rte_latencystats_init(1, NULL);
3934                 if (ret)
3935                         fprintf(stderr,
3936                                 "Warning: latencystats init() returned error %d\n",
3937                                 ret);
3938                 fprintf(stderr, "Latencystats running on lcore %d\n",
3939                         latencystats_lcore_id);
3940         }
3941 #endif
3942
3943         /* Setup bitrate stats */
3944 #ifdef RTE_LIB_BITRATESTATS
3945         if (bitrate_enabled != 0) {
3946                 bitrate_data = rte_stats_bitrate_create();
3947                 if (bitrate_data == NULL)
3948                         rte_exit(EXIT_FAILURE,
3949                                 "Could not allocate bitrate data.\n");
3950                 rte_stats_bitrate_reg(bitrate_data);
3951         }
3952 #endif
3953
3954 #ifdef RTE_LIB_CMDLINE
3955         if (strlen(cmdline_filename) != 0)
3956                 cmdline_read_from_file(cmdline_filename);
3957
3958         if (interactive == 1) {
3959                 if (auto_start) {
3960                         printf("Start automatic packet forwarding\n");
3961                         start_packet_forwarding(0);
3962                 }
3963                 prompt();
3964                 pmd_test_exit();
3965         } else
3966 #endif
3967         {
3968                 char c;
3969                 int rc;
3970
3971                 f_quit = 0;
3972
3973                 printf("No commandline core given, start packet forwarding\n");
3974                 start_packet_forwarding(tx_first);
3975                 if (stats_period != 0) {
3976                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3977                         uint64_t timer_period;
3978
3979                         /* Convert to number of cycles */
3980                         timer_period = stats_period * rte_get_timer_hz();
3981
3982                         while (f_quit == 0) {
3983                                 cur_time = rte_get_timer_cycles();
3984                                 diff_time += cur_time - prev_time;
3985
3986                                 if (diff_time >= timer_period) {
3987                                         print_stats();
3988                                         /* Reset the timer */
3989                                         diff_time = 0;
3990                                 }
3991                                 /* Sleep to avoid unnecessary checks */
3992                                 prev_time = cur_time;
3993                                 rte_delay_us_sleep(US_PER_S);
3994                         }
3995                 }
3996
3997                 printf("Press enter to exit\n");
3998                 rc = read(0, &c, 1);
3999                 pmd_test_exit();
4000                 if (rc < 0)
4001                         return 1;
4002         }
4003
4004         ret = rte_eal_cleanup();
4005         if (ret != 0)
4006                 rte_exit(EXIT_FAILURE,
4007                          "EAL cleanup failed: %s\n", strerror(-ret));
4008
4009         return EXIT_SUCCESS;
4010 }