app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_NET_IXGBE
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIB_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIB_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIB_LATENCYSTATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use main core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 219 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 220
 221 /*
 222  * Configuration of packet segments used by the "txonly" processing engine.
 223  */
 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 226         TXONLY_DEF_PACKET_LEN,
 227 };
 228 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 229
 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 231 /**< Split policy for packets to TX. */
 232
 233 uint8_t txonly_multi_flow;
 234 /**< Whether multiple flows are generated in TXONLY mode. */
 235
 236 uint32_t tx_pkt_times_inter;
 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 238
 239 uint32_t tx_pkt_times_intra;
 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 241
 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 245
 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 247 uint8_t dcb_config = 0;
 248
 249 /*
 250  * Configurable number of RX/TX queues.
 251  */
 252 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 253 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 254 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 255
 256 /*
 257  * Configurable number of RX/TX ring descriptors.
 258  * Defaults are supplied by drivers via ethdev.
 259  */
 260 #define RTE_TEST_RX_DESC_DEFAULT 0
 261 #define RTE_TEST_TX_DESC_DEFAULT 0
 262 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 263 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 264
 265 #define RTE_PMD_PARAM_UNSET -1
 266 /*
 267  * Configurable values of RX and TX ring threshold registers.
 268  */
 269
 270 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 271 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 272 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 273
 274 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 275 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 276 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of RX free threshold.
 280  */
 281 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of RX drop enable.
 285  */
 286 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of TX free threshold.
 290  */
 291 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 292
 293 /*
 294  * Configurable value of TX RS bit threshold.
 295  */
 296 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 297
 298 /*
 299  * Configurable value of buffered packets before sending.
 300  */
 301 uint16_t noisy_tx_sw_bufsz;
 302
 303 /*
 304  * Configurable value of packet buffer timeout.
 305  */
 306 uint16_t noisy_tx_sw_buf_flush_time;
 307
 308 /*
 309  * Configurable value for size of VNF internal memory area
 310  * used for simulating noisy neighbour behaviour
 311  */
 312 uint64_t noisy_lkup_mem_sz;
 313
 314 /*
 315  * Configurable value of number of random writes done in
 316  * VNF simulation memory area.
 317  */
 318 uint64_t noisy_lkup_num_writes;
 319
 320 /*
 321  * Configurable value of number of random reads done in
 322  * VNF simulation memory area.
 323  */
 324 uint64_t noisy_lkup_num_reads;
 325
 326 /*
 327  * Configurable value of number of random reads/writes done in
 328  * VNF simulation memory area.
 329  */
 330 uint64_t noisy_lkup_num_reads_writes;
 331
 332 /*
 333  * Receive Side Scaling (RSS) configuration.
 334  */
 335 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 336
 337 /*
 338  * Port topology configuration
 339  */
 340 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 341
 342 /*
 343  * Avoids to flush all the RX streams before starts forwarding.
 344  */
 345 uint8_t no_flush_rx = 0; /* flush by default */
 346
 347 /*
 348  * Flow API isolated mode.
 349  */
 350 uint8_t flow_isolate_all;
 351
 352 /*
 353  * Avoids to check link status when starting/stopping a port.
 354  */
 355 uint8_t no_link_check = 0; /* check by default */
 356
 357 /*
 358  * Don't automatically start all ports in interactive mode.
 359  */
 360 uint8_t no_device_start = 0;
 361
 362 /*
 363  * Enable link status change notification
 364  */
 365 uint8_t lsc_interrupt = 1; /* enabled by default */
 366
 367 /*
 368  * Enable device removal notification.
 369  */
 370 uint8_t rmv_interrupt = 1; /* enabled by default */
 371
 372 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 373
 374 /* After attach, port setup is called on event or by iterator */
 375 bool setup_on_probe_event = true;
 376
 377 /* Clear ptypes on port initialization. */
 378 uint8_t clear_ptypes = true;
 379
 380 /* Hairpin ports configuration mode. */
 381 uint16_t hairpin_mode;
 382
 383 /* Pretty printing of ethdev events */
 384 static const char * const eth_event_desc[] = {
 385         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 386         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 387         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 388         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 389         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 390         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 391         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 392         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 393         [RTE_ETH_EVENT_NEW] = "device probed",
 394         [RTE_ETH_EVENT_DESTROY] = "device released",
 395         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 396         [RTE_ETH_EVENT_MAX] = NULL,
 397 };
 398
 399 /*
 400  * Display or mask ether events
 401  * Default to all events except VF_MBOX
 402  */
 403 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 404                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 405                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 406                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 411 /*
 412  * Decide if all memory are locked for performance.
 413  */
 414 int do_mlockall = 0;
 415
 416 /*
 417  * NIC bypass mode configuration options.
 418  */
 419
 420 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 421 /* The NIC bypass watchdog timeout. */
 422 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 423 #endif
 424
 425
 426 #ifdef RTE_LIB_LATENCYSTATS
 427
 428 /*
 429  * Set when latency stats is enabled in the commandline
 430  */
 431 uint8_t latencystats_enabled;
 432
 433 /*
 434  * Lcore ID to serive latency statistics.
 435  */
 436 lcoreid_t latencystats_lcore_id = -1;
 437
 438 #endif
 439
 440 /*
 441  * Ethernet device configuration.
 442  */
 443 struct rte_eth_rxmode rx_mode = {
 444         /* Default maximum frame length.
 445          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 446          * in init_config().
 447          */
 448         .max_rx_pkt_len = 0,
 449 };
 450
 451 struct rte_eth_txmode tx_mode = {
 452         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 453 };
 454
 455 struct rte_fdir_conf fdir_conf = {
 456         .mode = RTE_FDIR_MODE_NONE,
 457         .pballoc = RTE_FDIR_PBALLOC_64K,
 458         .status = RTE_FDIR_REPORT_STATUS,
 459         .mask = {
 460                 .vlan_tci_mask = 0xFFEF,
 461                 .ipv4_mask     = {
 462                         .src_ip = 0xFFFFFFFF,
 463                         .dst_ip = 0xFFFFFFFF,
 464                 },
 465                 .ipv6_mask     = {
 466                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 467                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 468                 },
 469                 .src_port_mask = 0xFFFF,
 470                 .dst_port_mask = 0xFFFF,
 471                 .mac_addr_byte_mask = 0xFF,
 472                 .tunnel_type_mask = 1,
 473                 .tunnel_id_mask = 0xFFFFFFFF,
 474         },
 475         .drop_queue = 127,
 476 };
 477
 478 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 479
 480 /*
 481  * Display zero values by default for xstats
 482  */
 483 uint8_t xstats_hide_zero;
 484
 485 /*
 486  * Measure of CPU cycles disabled by default
 487  */
 488 uint8_t record_core_cycles;
 489
 490 /*
 491  * Display of RX and TX bursts disabled by default
 492  */
 493 uint8_t record_burst_stats;
 494
 495 unsigned int num_sockets = 0;
 496 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 497
 498 #ifdef RTE_LIB_BITRATESTATS
 499 /* Bitrate statistics */
 500 struct rte_stats_bitrates *bitrate_data;
 501 lcoreid_t bitrate_lcore_id;
 502 uint8_t bitrate_enabled;
 503 #endif
 504
 505 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 506 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 507
 508 /*
 509  * hexadecimal bitmask of RX mq mode can be enabled.
 510  */
 511 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 512
 513 /*
 514  * Used to set forced link speed
 515  */
 516 uint32_t eth_link_speed;
 517
 518 /* Forward function declarations */
 519 static void setup_attached_port(portid_t pi);
 520 static void check_all_ports_link_status(uint32_t port_mask);
 521 static int eth_event_callback(portid_t port_id,
 522                               enum rte_eth_event_type type,
 523                               void *param, void *ret_param);
 524 static void dev_event_callback(const char *device_name,
 525                                 enum rte_dev_event_type type,
 526                                 void *param);
 527
 528 /*
 529  * Check if all the ports are started.
 530  * If yes, return positive value. If not, return zero.
 531  */
 532 static int all_ports_started(void);
 533
 534 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 535 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 536
 537 /* Holds the registered mbuf dynamic flags names. */
 538 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 539
 540 /*
 541  * Helper function to check if socket is already discovered.
 542  * If yes, return positive value. If not, return zero.
 543  */
 544 int
 545 new_socket_id(unsigned int socket_id)
 546 {
 547         unsigned int i;
 548
 549         for (i = 0; i < num_sockets; i++) {
 550                 if (socket_ids[i] == socket_id)
 551                         return 0;
 552         }
 553         return 1;
 554 }
 555
 556 /*
 557  * Setup default configuration.
 558  */
 559 static void
 560 set_default_fwd_lcores_config(void)
 561 {
 562         unsigned int i;
 563         unsigned int nb_lc;
 564         unsigned int sock_num;
 565
 566         nb_lc = 0;
 567         for (i = 0; i < RTE_MAX_LCORE; i++) {
 568                 if (!rte_lcore_is_enabled(i))
 569                         continue;
 570                 sock_num = rte_lcore_to_socket_id(i);
 571                 if (new_socket_id(sock_num)) {
 572                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 573                                 rte_exit(EXIT_FAILURE,
 574                                          "Total sockets greater than %u\n",
 575                                          RTE_MAX_NUMA_NODES);
 576                         }
 577                         socket_ids[num_sockets++] = sock_num;
 578                 }
 579                 if (i == rte_get_main_lcore())
 580                         continue;
 581                 fwd_lcores_cpuids[nb_lc++] = i;
 582         }
 583         nb_lcores = (lcoreid_t) nb_lc;
 584         nb_cfg_lcores = nb_lcores;
 585         nb_fwd_lcores = 1;
 586 }
 587
 588 static void
 589 set_def_peer_eth_addrs(void)
 590 {
 591         portid_t i;
 592
 593         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 594                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 595                 peer_eth_addrs[i].addr_bytes[5] = i;
 596         }
 597 }
 598
 599 static void
 600 set_default_fwd_ports_config(void)
 601 {
 602         portid_t pt_id;
 603         int i = 0;
 604
 605         RTE_ETH_FOREACH_DEV(pt_id) {
 606                 fwd_ports_ids[i++] = pt_id;
 607
 608                 /* Update sockets info according to the attached device */
 609                 int socket_id = rte_eth_dev_socket_id(pt_id);
 610                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 611                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 612                                 rte_exit(EXIT_FAILURE,
 613                                          "Total sockets greater than %u\n",
 614                                          RTE_MAX_NUMA_NODES);
 615                         }
 616                         socket_ids[num_sockets++] = socket_id;
 617                 }
 618         }
 619
 620         nb_cfg_ports = nb_ports;
 621         nb_fwd_ports = nb_ports;
 622 }
 623
 624 void
 625 set_def_fwd_config(void)
 626 {
 627         set_default_fwd_lcores_config();
 628         set_def_peer_eth_addrs();
 629         set_default_fwd_ports_config();
 630 }
 631
 632 /* extremely pessimistic estimation of memory required to create a mempool */
 633 static int
 634 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 635 {
 636         unsigned int n_pages, mbuf_per_pg, leftover;
 637         uint64_t total_mem, mbuf_mem, obj_sz;
 638
 639         /* there is no good way to predict how much space the mempool will
 640          * occupy because it will allocate chunks on the fly, and some of those
 641          * will come from default DPDK memory while some will come from our
 642          * external memory, so just assume 128MB will be enough for everyone.
 643          */
 644         uint64_t hdr_mem = 128 << 20;
 645
 646         /* account for possible non-contiguousness */
 647         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 648         if (obj_sz > pgsz) {
 649                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 650                 return -1;
 651         }
 652
 653         mbuf_per_pg = pgsz / obj_sz;
 654         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 655         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 656
 657         mbuf_mem = n_pages * pgsz;
 658
 659         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 660
 661         if (total_mem > SIZE_MAX) {
 662                 TESTPMD_LOG(ERR, "Memory size too big\n");
 663                 return -1;
 664         }
 665         *out = (size_t)total_mem;
 666
 667         return 0;
 668 }
 669
 670 static int
 671 pagesz_flags(uint64_t page_sz)
 672 {
 673         /* as per mmap() manpage, all page sizes are log2 of page size
 674          * shifted by MAP_HUGE_SHIFT
 675          */
 676         int log2 = rte_log2_u64(page_sz);
 677
 678         return (log2 << HUGE_SHIFT);
 679 }
 680
 681 static void *
 682 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 683 {
 684         void *addr;
 685         int flags;
 686
 687         /* allocate anonymous hugepages */
 688         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 689         if (huge)
 690                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 691
 692         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 693         if (addr == MAP_FAILED)
 694                 return NULL;
 695
 696         return addr;
 697 }
 698
 699 struct extmem_param {
 700         void *addr;
 701         size_t len;
 702         size_t pgsz;
 703         rte_iova_t *iova_table;
 704         unsigned int iova_table_len;
 705 };
 706
 707 static int
 708 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 709                 bool huge)
 710 {
 711         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 712                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 713         unsigned int cur_page, n_pages, pgsz_idx;
 714         size_t mem_sz, cur_pgsz;
 715         rte_iova_t *iovas = NULL;
 716         void *addr;
 717         int ret;
 718
 719         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 720                 /* skip anything that is too big */
 721                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 722                         continue;
 723
 724                 cur_pgsz = pgsizes[pgsz_idx];
 725
 726                 /* if we were told not to allocate hugepages, override */
 727                 if (!huge)
 728                         cur_pgsz = sysconf(_SC_PAGESIZE);
 729
 730                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 731                 if (ret < 0) {
 732                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 733                         return -1;
 734                 }
 735
 736                 /* allocate our memory */
 737                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 738
 739                 /* if we couldn't allocate memory with a specified page size,
 740                  * that doesn't mean we can't do it with other page sizes, so
 741                  * try another one.
 742                  */
 743                 if (addr == NULL)
 744                         continue;
 745
 746                 /* store IOVA addresses for every page in this memory area */
 747                 n_pages = mem_sz / cur_pgsz;
 748
 749                 iovas = malloc(sizeof(*iovas) * n_pages);
 750
 751                 if (iovas == NULL) {
 752                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 753                         goto fail;
 754                 }
 755                 /* lock memory if it's not huge pages */
 756                 if (!huge)
 757                         mlock(addr, mem_sz);
 758
 759                 /* populate IOVA addresses */
 760                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 761                         rte_iova_t iova;
 762                         size_t offset;
 763                         void *cur;
 764
 765                         offset = cur_pgsz * cur_page;
 766                         cur = RTE_PTR_ADD(addr, offset);
 767
 768                         /* touch the page before getting its IOVA */
 769                         *(volatile char *)cur = 0;
 770
 771                         iova = rte_mem_virt2iova(cur);
 772
 773                         iovas[cur_page] = iova;
 774                 }
 775
 776                 break;
 777         }
 778         /* if we couldn't allocate anything */
 779         if (iovas == NULL)
 780                 return -1;
 781
 782         param->addr = addr;
 783         param->len = mem_sz;
 784         param->pgsz = cur_pgsz;
 785         param->iova_table = iovas;
 786         param->iova_table_len = n_pages;
 787
 788         return 0;
 789 fail:
 790         if (iovas)
 791                 free(iovas);
 792         if (addr)
 793                 munmap(addr, mem_sz);
 794
 795         return -1;
 796 }
 797
 798 static int
 799 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 800 {
 801         struct extmem_param param;
 802         int socket_id, ret;
 803
 804         memset(&param, 0, sizeof(param));
 805
 806         /* check if our heap exists */
 807         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 808         if (socket_id < 0) {
 809                 /* create our heap */
 810                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 811                 if (ret < 0) {
 812                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 813                         return -1;
 814                 }
 815         }
 816
 817         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 818         if (ret < 0) {
 819                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 820                 return -1;
 821         }
 822
 823         /* we now have a valid memory area, so add it to heap */
 824         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 825                         param.addr, param.len, param.iova_table,
 826                         param.iova_table_len, param.pgsz);
 827
 828         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 829
 830         /* not needed any more */
 831         free(param.iova_table);
 832
 833         if (ret < 0) {
 834                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 835                 munmap(param.addr, param.len);
 836                 return -1;
 837         }
 838
 839         /* success */
 840
 841         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 842                         param.len >> 20);
 843
 844         return 0;
 845 }
 846 static void
 847 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 848              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 849 {
 850         uint16_t pid = 0;
 851         int ret;
 852
 853         RTE_ETH_FOREACH_DEV(pid) {
 854                 struct rte_eth_dev *dev =
 855                         &rte_eth_devices[pid];
 856
 857                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 858                                         memhdr->len);
 859                 if (ret) {
 860                         TESTPMD_LOG(DEBUG,
 861                                     "unable to DMA unmap addr 0x%p "
 862                                     "for device %s\n",
 863                                     memhdr->addr, dev->data->name);
 864                 }
 865         }
 866         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 867         if (ret) {
 868                 TESTPMD_LOG(DEBUG,
 869                             "unable to un-register addr 0x%p\n", memhdr->addr);
 870         }
 871 }
 872
 873 static void
 874 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 875            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 876 {
 877         uint16_t pid = 0;
 878         size_t page_size = sysconf(_SC_PAGESIZE);
 879         int ret;
 880
 881         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 882                                   page_size);
 883         if (ret) {
 884                 TESTPMD_LOG(DEBUG,
 885                             "unable to register addr 0x%p\n", memhdr->addr);
 886                 return;
 887         }
 888         RTE_ETH_FOREACH_DEV(pid) {
 889                 struct rte_eth_dev *dev =
 890                         &rte_eth_devices[pid];
 891
 892                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 893                                       memhdr->len);
 894                 if (ret) {
 895                         TESTPMD_LOG(DEBUG,
 896                                     "unable to DMA map addr 0x%p "
 897                                     "for device %s\n",
 898                                     memhdr->addr, dev->data->name);
 899                 }
 900         }
 901 }
 902
 903 static unsigned int
 904 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 905             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 906 {
 907         struct rte_pktmbuf_extmem *xmem;
 908         unsigned int ext_num, zone_num, elt_num;
 909         uint16_t elt_size;
 910
 911         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 912         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 913         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 914
 915         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 916         if (xmem == NULL) {
 917                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 918                                  "external buffer descriptors\n");
 919                 *ext_mem = NULL;
 920                 return 0;
 921         }
 922         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 923                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 924                 const struct rte_memzone *mz;
 925                 char mz_name[RTE_MEMZONE_NAMESIZE];
 926                 int ret;
 927
 928                 ret = snprintf(mz_name, sizeof(mz_name),
 929                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 930                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 931                         errno = ENAMETOOLONG;
 932                         ext_num = 0;
 933                         break;
 934                 }
 935                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 936                                                  socket_id,
 937                                                  RTE_MEMZONE_IOVA_CONTIG |
 938                                                  RTE_MEMZONE_1GB |
 939                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 940                                                  EXTBUF_ZONE_SIZE);
 941                 if (mz == NULL) {
 942                         /*
 943                          * The caller exits on external buffer creation
 944                          * error, so there is no need to free memzones.
 945                          */
 946                         errno = ENOMEM;
 947                         ext_num = 0;
 948                         break;
 949                 }
 950                 xseg->buf_ptr = mz->addr;
 951                 xseg->buf_iova = mz->iova;
 952                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 953                 xseg->elt_size = elt_size;
 954         }
 955         if (ext_num == 0 && xmem != NULL) {
 956                 free(xmem);
 957                 xmem = NULL;
 958         }
 959         *ext_mem = xmem;
 960         return ext_num;
 961 }
 962
 963 /*
 964  * Configuration initialisation done once at init time.
 965  */
 966 static struct rte_mempool *
 967 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 968                  unsigned int socket_id, uint16_t size_idx)
 969 {
 970         char pool_name[RTE_MEMPOOL_NAMESIZE];
 971         struct rte_mempool *rte_mp = NULL;
 972         uint32_t mb_size;
 973
 974         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 975         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 976
 977         TESTPMD_LOG(INFO,
 978                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 979                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 980
 981         switch (mp_alloc_type) {
 982         case MP_ALLOC_NATIVE:
 983                 {
 984                         /* wrapper to rte_mempool_create() */
 985                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 986                                         rte_mbuf_best_mempool_ops());
 987                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 988                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 989                         break;
 990                 }
 991         case MP_ALLOC_ANON:
 992                 {
 993                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 994                                 mb_size, (unsigned int) mb_mempool_cache,
 995                                 sizeof(struct rte_pktmbuf_pool_private),
 996                                 socket_id, mempool_flags);
 997                         if (rte_mp == NULL)
 998                                 goto err;
 999
1000                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1001                                 rte_mempool_free(rte_mp);
1002                                 rte_mp = NULL;
1003                                 goto err;
1004                         }
1005                         rte_pktmbuf_pool_init(rte_mp, NULL);
1006                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1007                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1008                         break;
1009                 }
1010         case MP_ALLOC_XMEM:
1011         case MP_ALLOC_XMEM_HUGE:
1012                 {
1013                         int heap_socket;
1014                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1015
1016                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1017                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1018
1019                         heap_socket =
1020                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1021                         if (heap_socket < 0)
1022                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1023
1024                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1025                                         rte_mbuf_best_mempool_ops());
1026                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1027                                         mb_mempool_cache, 0, mbuf_seg_size,
1028                                         heap_socket);
1029                         break;
1030                 }
1031         case MP_ALLOC_XBUF:
1032                 {
1033                         struct rte_pktmbuf_extmem *ext_mem;
1034                         unsigned int ext_num;
1035
1036                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1037                                                socket_id, pool_name, &ext_mem);
1038                         if (ext_num == 0)
1039                                 rte_exit(EXIT_FAILURE,
1040                                          "Can't create pinned data buffers\n");
1041
1042                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1043                                         rte_mbuf_best_mempool_ops());
1044                         rte_mp = rte_pktmbuf_pool_create_extbuf
1045                                         (pool_name, nb_mbuf, mb_mempool_cache,
1046                                          0, mbuf_seg_size, socket_id,
1047                                          ext_mem, ext_num);
1048                         free(ext_mem);
1049                         break;
1050                 }
1051         default:
1052                 {
1053                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1054                 }
1055         }
1056
1057 err:
1058         if (rte_mp == NULL) {
1059                 rte_exit(EXIT_FAILURE,
1060                         "Creation of mbuf pool for socket %u failed: %s\n",
1061                         socket_id, rte_strerror(rte_errno));
1062         } else if (verbose_level > 0) {
1063                 rte_mempool_dump(stdout, rte_mp);
1064         }
1065         return rte_mp;
1066 }
1067
1068 /*
1069  * Check given socket id is valid or not with NUMA mode,
1070  * if valid, return 0, else return -1
1071  */
1072 static int
1073 check_socket_id(const unsigned int socket_id)
1074 {
1075         static int warning_once = 0;
1076
1077         if (new_socket_id(socket_id)) {
1078                 if (!warning_once && numa_support)
1079                         printf("Warning: NUMA should be configured manually by"
1080                                " using --port-numa-config and"
1081                                " --ring-numa-config parameters along with"
1082                                " --numa.\n");
1083                 warning_once = 1;
1084                 return -1;
1085         }
1086         return 0;
1087 }
1088
1089 /*
1090  * Get the allowed maximum number of RX queues.
1091  * *pid return the port id which has minimal value of
1092  * max_rx_queues in all ports.
1093  */
1094 queueid_t
1095 get_allowed_max_nb_rxq(portid_t *pid)
1096 {
1097         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1098         bool max_rxq_valid = false;
1099         portid_t pi;
1100         struct rte_eth_dev_info dev_info;
1101
1102         RTE_ETH_FOREACH_DEV(pi) {
1103                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1104                         continue;
1105
1106                 max_rxq_valid = true;
1107                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1108                         allowed_max_rxq = dev_info.max_rx_queues;
1109                         *pid = pi;
1110                 }
1111         }
1112         return max_rxq_valid ? allowed_max_rxq : 0;
1113 }
1114
1115 /*
1116  * Check input rxq is valid or not.
1117  * If input rxq is not greater than any of maximum number
1118  * of RX queues of all ports, it is valid.
1119  * if valid, return 0, else return -1
1120  */
1121 int
1122 check_nb_rxq(queueid_t rxq)
1123 {
1124         queueid_t allowed_max_rxq;
1125         portid_t pid = 0;
1126
1127         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1128         if (rxq > allowed_max_rxq) {
1129                 printf("Fail: input rxq (%u) can't be greater "
1130                        "than max_rx_queues (%u) of port %u\n",
1131                        rxq,
1132                        allowed_max_rxq,
1133                        pid);
1134                 return -1;
1135         }
1136         return 0;
1137 }
1138
1139 /*
1140  * Get the allowed maximum number of TX queues.
1141  * *pid return the port id which has minimal value of
1142  * max_tx_queues in all ports.
1143  */
1144 queueid_t
1145 get_allowed_max_nb_txq(portid_t *pid)
1146 {
1147         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1148         bool max_txq_valid = false;
1149         portid_t pi;
1150         struct rte_eth_dev_info dev_info;
1151
1152         RTE_ETH_FOREACH_DEV(pi) {
1153                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1154                         continue;
1155
1156                 max_txq_valid = true;
1157                 if (dev_info.max_tx_queues < allowed_max_txq) {
1158                         allowed_max_txq = dev_info.max_tx_queues;
1159                         *pid = pi;
1160                 }
1161         }
1162         return max_txq_valid ? allowed_max_txq : 0;
1163 }
1164
1165 /*
1166  * Check input txq is valid or not.
1167  * If input txq is not greater than any of maximum number
1168  * of TX queues of all ports, it is valid.
1169  * if valid, return 0, else return -1
1170  */
1171 int
1172 check_nb_txq(queueid_t txq)
1173 {
1174         queueid_t allowed_max_txq;
1175         portid_t pid = 0;
1176
1177         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1178         if (txq > allowed_max_txq) {
1179                 printf("Fail: input txq (%u) can't be greater "
1180                        "than max_tx_queues (%u) of port %u\n",
1181                        txq,
1182                        allowed_max_txq,
1183                        pid);
1184                 return -1;
1185         }
1186         return 0;
1187 }
1188
1189 /*
1190  * Get the allowed maximum number of RXDs of every rx queue.
1191  * *pid return the port id which has minimal value of
1192  * max_rxd in all queues of all ports.
1193  */
1194 static uint16_t
1195 get_allowed_max_nb_rxd(portid_t *pid)
1196 {
1197         uint16_t allowed_max_rxd = UINT16_MAX;
1198         portid_t pi;
1199         struct rte_eth_dev_info dev_info;
1200
1201         RTE_ETH_FOREACH_DEV(pi) {
1202                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1203                         continue;
1204
1205                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1206                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1207                         *pid = pi;
1208                 }
1209         }
1210         return allowed_max_rxd;
1211 }
1212
1213 /*
1214  * Get the allowed minimal number of RXDs of every rx queue.
1215  * *pid return the port id which has minimal value of
1216  * min_rxd in all queues of all ports.
1217  */
1218 static uint16_t
1219 get_allowed_min_nb_rxd(portid_t *pid)
1220 {
1221         uint16_t allowed_min_rxd = 0;
1222         portid_t pi;
1223         struct rte_eth_dev_info dev_info;
1224
1225         RTE_ETH_FOREACH_DEV(pi) {
1226                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1227                         continue;
1228
1229                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1230                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1231                         *pid = pi;
1232                 }
1233         }
1234
1235         return allowed_min_rxd;
1236 }
1237
1238 /*
1239  * Check input rxd is valid or not.
1240  * If input rxd is not greater than any of maximum number
1241  * of RXDs of every Rx queues and is not less than any of
1242  * minimal number of RXDs of every Rx queues, it is valid.
1243  * if valid, return 0, else return -1
1244  */
1245 int
1246 check_nb_rxd(queueid_t rxd)
1247 {
1248         uint16_t allowed_max_rxd;
1249         uint16_t allowed_min_rxd;
1250         portid_t pid = 0;
1251
1252         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1253         if (rxd > allowed_max_rxd) {
1254                 printf("Fail: input rxd (%u) can't be greater "
1255                        "than max_rxds (%u) of port %u\n",
1256                        rxd,
1257                        allowed_max_rxd,
1258                        pid);
1259                 return -1;
1260         }
1261
1262         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1263         if (rxd < allowed_min_rxd) {
1264                 printf("Fail: input rxd (%u) can't be less "
1265                        "than min_rxds (%u) of port %u\n",
1266                        rxd,
1267                        allowed_min_rxd,
1268                        pid);
1269                 return -1;
1270         }
1271
1272         return 0;
1273 }
1274
1275 /*
1276  * Get the allowed maximum number of TXDs of every rx queues.
1277  * *pid return the port id which has minimal value of
1278  * max_txd in every tx queue.
1279  */
1280 static uint16_t
1281 get_allowed_max_nb_txd(portid_t *pid)
1282 {
1283         uint16_t allowed_max_txd = UINT16_MAX;
1284         portid_t pi;
1285         struct rte_eth_dev_info dev_info;
1286
1287         RTE_ETH_FOREACH_DEV(pi) {
1288                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1289                         continue;
1290
1291                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1292                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1293                         *pid = pi;
1294                 }
1295         }
1296         return allowed_max_txd;
1297 }
1298
1299 /*
1300  * Get the allowed maximum number of TXDs of every tx queues.
1301  * *pid return the port id which has minimal value of
1302  * min_txd in every tx queue.
1303  */
1304 static uint16_t
1305 get_allowed_min_nb_txd(portid_t *pid)
1306 {
1307         uint16_t allowed_min_txd = 0;
1308         portid_t pi;
1309         struct rte_eth_dev_info dev_info;
1310
1311         RTE_ETH_FOREACH_DEV(pi) {
1312                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1313                         continue;
1314
1315                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1316                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1317                         *pid = pi;
1318                 }
1319         }
1320
1321         return allowed_min_txd;
1322 }
1323
1324 /*
1325  * Check input txd is valid or not.
1326  * If input txd is not greater than any of maximum number
1327  * of TXDs of every Rx queues, it is valid.
1328  * if valid, return 0, else return -1
1329  */
1330 int
1331 check_nb_txd(queueid_t txd)
1332 {
1333         uint16_t allowed_max_txd;
1334         uint16_t allowed_min_txd;
1335         portid_t pid = 0;
1336
1337         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1338         if (txd > allowed_max_txd) {
1339                 printf("Fail: input txd (%u) can't be greater "
1340                        "than max_txds (%u) of port %u\n",
1341                        txd,
1342                        allowed_max_txd,
1343                        pid);
1344                 return -1;
1345         }
1346
1347         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1348         if (txd < allowed_min_txd) {
1349                 printf("Fail: input txd (%u) can't be less "
1350                        "than min_txds (%u) of port %u\n",
1351                        txd,
1352                        allowed_min_txd,
1353                        pid);
1354                 return -1;
1355         }
1356         return 0;
1357 }
1358
1359
1360 /*
1361  * Get the allowed maximum number of hairpin queues.
1362  * *pid return the port id which has minimal value of
1363  * max_hairpin_queues in all ports.
1364  */
1365 queueid_t
1366 get_allowed_max_nb_hairpinq(portid_t *pid)
1367 {
1368         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1369         portid_t pi;
1370         struct rte_eth_hairpin_cap cap;
1371
1372         RTE_ETH_FOREACH_DEV(pi) {
1373                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1374                         *pid = pi;
1375                         return 0;
1376                 }
1377                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1378                         allowed_max_hairpinq = cap.max_nb_queues;
1379                         *pid = pi;
1380                 }
1381         }
1382         return allowed_max_hairpinq;
1383 }
1384
1385 /*
1386  * Check input hairpin is valid or not.
1387  * If input hairpin is not greater than any of maximum number
1388  * of hairpin queues of all ports, it is valid.
1389  * if valid, return 0, else return -1
1390  */
1391 int
1392 check_nb_hairpinq(queueid_t hairpinq)
1393 {
1394         queueid_t allowed_max_hairpinq;
1395         portid_t pid = 0;
1396
1397         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1398         if (hairpinq > allowed_max_hairpinq) {
1399                 printf("Fail: input hairpin (%u) can't be greater "
1400                        "than max_hairpin_queues (%u) of port %u\n",
1401                        hairpinq, allowed_max_hairpinq, pid);
1402                 return -1;
1403         }
1404         return 0;
1405 }
1406
1407 static void
1408 init_config(void)
1409 {
1410         portid_t pid;
1411         struct rte_port *port;
1412         struct rte_mempool *mbp;
1413         unsigned int nb_mbuf_per_pool;
1414         lcoreid_t  lc_id;
1415         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1416         struct rte_gro_param gro_param;
1417         uint32_t gso_types;
1418         uint16_t data_size;
1419         bool warning = 0;
1420         int k;
1421         int ret;
1422
1423         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1424
1425         /* Configuration of logical cores. */
1426         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1427                                 sizeof(struct fwd_lcore *) * nb_lcores,
1428                                 RTE_CACHE_LINE_SIZE);
1429         if (fwd_lcores == NULL) {
1430                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1431                                                         "failed\n", nb_lcores);
1432         }
1433         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1434                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1435                                                sizeof(struct fwd_lcore),
1436                                                RTE_CACHE_LINE_SIZE);
1437                 if (fwd_lcores[lc_id] == NULL) {
1438                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1439                                                                 "failed\n");
1440                 }
1441                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1442         }
1443
1444         RTE_ETH_FOREACH_DEV(pid) {
1445                 port = &ports[pid];
1446                 /* Apply default TxRx configuration for all ports */
1447                 port->dev_conf.txmode = tx_mode;
1448                 port->dev_conf.rxmode = rx_mode;
1449
1450                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1451                 if (ret != 0)
1452                         rte_exit(EXIT_FAILURE,
1453                                  "rte_eth_dev_info_get() failed\n");
1454
1455                 ret = update_jumbo_frame_offload(pid);
1456                 if (ret != 0)
1457                         printf("Updating jumbo frame offload failed for port %u\n",
1458                                 pid);
1459
1460                 if (!(port->dev_info.tx_offload_capa &
1461                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1462                         port->dev_conf.txmode.offloads &=
1463                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1464                 if (numa_support) {
1465                         if (port_numa[pid] != NUMA_NO_CONFIG)
1466                                 port_per_socket[port_numa[pid]]++;
1467                         else {
1468                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1469
1470                                 /*
1471                                  * if socket_id is invalid,
1472                                  * set to the first available socket.
1473                                  */
1474                                 if (check_socket_id(socket_id) < 0)
1475                                         socket_id = socket_ids[0];
1476                                 port_per_socket[socket_id]++;
1477                         }
1478                 }
1479
1480                 /* Apply Rx offloads configuration */
1481                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1482                         port->rx_conf[k].offloads =
1483                                 port->dev_conf.rxmode.offloads;
1484                 /* Apply Tx offloads configuration */
1485                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1486                         port->tx_conf[k].offloads =
1487                                 port->dev_conf.txmode.offloads;
1488
1489                 if (eth_link_speed)
1490                         port->dev_conf.link_speeds = eth_link_speed;
1491
1492                 /* set flag to initialize port/queue */
1493                 port->need_reconfig = 1;
1494                 port->need_reconfig_queues = 1;
1495                 port->tx_metadata = 0;
1496
1497                 /* Check for maximum number of segments per MTU. Accordingly
1498                  * update the mbuf data size.
1499                  */
1500                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1501                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1502                         data_size = rx_mode.max_rx_pkt_len /
1503                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1504
1505                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1506                                                         mbuf_data_size[0]) {
1507                                 mbuf_data_size[0] = data_size +
1508                                                  RTE_PKTMBUF_HEADROOM;
1509                                 warning = 1;
1510                         }
1511                 }
1512         }
1513
1514         if (warning)
1515                 TESTPMD_LOG(WARNING,
1516                             "Configured mbuf size of the first segment %hu\n",
1517                             mbuf_data_size[0]);
1518         /*
1519          * Create pools of mbuf.
1520          * If NUMA support is disabled, create a single pool of mbuf in
1521          * socket 0 memory by default.
1522          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1523          *
1524          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1525          * nb_txd can be configured at run time.
1526          */
1527         if (param_total_num_mbufs)
1528                 nb_mbuf_per_pool = param_total_num_mbufs;
1529         else {
1530                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1531                         (nb_lcores * mb_mempool_cache) +
1532                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1533                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1534         }
1535
1536         if (numa_support) {
1537                 uint8_t i, j;
1538
1539                 for (i = 0; i < num_sockets; i++)
1540                         for (j = 0; j < mbuf_data_size_n; j++)
1541                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1542                                         mbuf_pool_create(mbuf_data_size[j],
1543                                                           nb_mbuf_per_pool,
1544                                                           socket_ids[i], j);
1545         } else {
1546                 uint8_t i;
1547
1548                 for (i = 0; i < mbuf_data_size_n; i++)
1549                         mempools[i] = mbuf_pool_create
1550                                         (mbuf_data_size[i],
1551                                          nb_mbuf_per_pool,
1552                                          socket_num == UMA_NO_CONFIG ?
1553                                          0 : socket_num, i);
1554         }
1555
1556         init_port_config();
1557
1558         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1559                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1560         /*
1561          * Records which Mbuf pool to use by each logical core, if needed.
1562          */
1563         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1564                 mbp = mbuf_pool_find(
1565                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1566
1567                 if (mbp == NULL)
1568                         mbp = mbuf_pool_find(0, 0);
1569                 fwd_lcores[lc_id]->mbp = mbp;
1570                 /* initialize GSO context */
1571                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1572                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1573                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1574                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1575                         RTE_ETHER_CRC_LEN;
1576                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1577         }
1578
1579         /* Configuration of packet forwarding streams. */
1580         if (init_fwd_streams() < 0)
1581                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1582
1583         fwd_config_setup();
1584
1585         /* create a gro context for each lcore */
1586         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1587         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1588         gro_param.max_item_per_flow = MAX_PKT_BURST;
1589         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1590                 gro_param.socket_id = rte_lcore_to_socket_id(
1591                                 fwd_lcores_cpuids[lc_id]);
1592                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1593                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1594                         rte_exit(EXIT_FAILURE,
1595                                         "rte_gro_ctx_create() failed\n");
1596                 }
1597         }
1598 }
1599
1600
1601 void
1602 reconfig(portid_t new_port_id, unsigned socket_id)
1603 {
1604         struct rte_port *port;
1605         int ret;
1606
1607         /* Reconfiguration of Ethernet ports. */
1608         port = &ports[new_port_id];
1609
1610         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1611         if (ret != 0)
1612                 return;
1613
1614         /* set flag to initialize port/queue */
1615         port->need_reconfig = 1;
1616         port->need_reconfig_queues = 1;
1617         port->socket_id = socket_id;
1618
1619         init_port_config();
1620 }
1621
1622
1623 int
1624 init_fwd_streams(void)
1625 {
1626         portid_t pid;
1627         struct rte_port *port;
1628         streamid_t sm_id, nb_fwd_streams_new;
1629         queueid_t q;
1630
1631         /* set socket id according to numa or not */
1632         RTE_ETH_FOREACH_DEV(pid) {
1633                 port = &ports[pid];
1634                 if (nb_rxq > port->dev_info.max_rx_queues) {
1635                         printf("Fail: nb_rxq(%d) is greater than "
1636                                 "max_rx_queues(%d)\n", nb_rxq,
1637                                 port->dev_info.max_rx_queues);
1638                         return -1;
1639                 }
1640                 if (nb_txq > port->dev_info.max_tx_queues) {
1641                         printf("Fail: nb_txq(%d) is greater than "
1642                                 "max_tx_queues(%d)\n", nb_txq,
1643                                 port->dev_info.max_tx_queues);
1644                         return -1;
1645                 }
1646                 if (numa_support) {
1647                         if (port_numa[pid] != NUMA_NO_CONFIG)
1648                                 port->socket_id = port_numa[pid];
1649                         else {
1650                                 port->socket_id = rte_eth_dev_socket_id(pid);
1651
1652                                 /*
1653                                  * if socket_id is invalid,
1654                                  * set to the first available socket.
1655                                  */
1656                                 if (check_socket_id(port->socket_id) < 0)
1657                                         port->socket_id = socket_ids[0];
1658                         }
1659                 }
1660                 else {
1661                         if (socket_num == UMA_NO_CONFIG)
1662                                 port->socket_id = 0;
1663                         else
1664                                 port->socket_id = socket_num;
1665                 }
1666         }
1667
1668         q = RTE_MAX(nb_rxq, nb_txq);
1669         if (q == 0) {
1670                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1671                 return -1;
1672         }
1673         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1674         if (nb_fwd_streams_new == nb_fwd_streams)
1675                 return 0;
1676         /* clear the old */
1677         if (fwd_streams != NULL) {
1678                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1679                         if (fwd_streams[sm_id] == NULL)
1680                                 continue;
1681                         rte_free(fwd_streams[sm_id]);
1682                         fwd_streams[sm_id] = NULL;
1683                 }
1684                 rte_free(fwd_streams);
1685                 fwd_streams = NULL;
1686         }
1687
1688         /* init new */
1689         nb_fwd_streams = nb_fwd_streams_new;
1690         if (nb_fwd_streams) {
1691                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1692                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1693                         RTE_CACHE_LINE_SIZE);
1694                 if (fwd_streams == NULL)
1695                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1696                                  " (struct fwd_stream *)) failed\n",
1697                                  nb_fwd_streams);
1698
1699                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1700                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1701                                 " struct fwd_stream", sizeof(struct fwd_stream),
1702                                 RTE_CACHE_LINE_SIZE);
1703                         if (fwd_streams[sm_id] == NULL)
1704                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1705                                          "(struct fwd_stream) failed\n");
1706                 }
1707         }
1708
1709         return 0;
1710 }
1711
1712 static void
1713 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1714 {
1715         uint64_t total_burst, sburst;
1716         uint64_t nb_burst;
1717         uint64_t burst_stats[4];
1718         uint16_t pktnb_stats[4];
1719         uint16_t nb_pkt;
1720         int burst_percent[4], sburstp;
1721         int i;
1722
1723         /*
1724          * First compute the total number of packet bursts and the
1725          * two highest numbers of bursts of the same number of packets.
1726          */
1727         memset(&burst_stats, 0x0, sizeof(burst_stats));
1728         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1729
1730         /* Show stats for 0 burst size always */
1731         total_burst = pbs->pkt_burst_spread[0];
1732         burst_stats[0] = pbs->pkt_burst_spread[0];
1733         pktnb_stats[0] = 0;
1734
1735         /* Find the next 2 burst sizes with highest occurrences. */
1736         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1737                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1738
1739                 if (nb_burst == 0)
1740                         continue;
1741
1742                 total_burst += nb_burst;
1743
1744                 if (nb_burst > burst_stats[1]) {
1745                         burst_stats[2] = burst_stats[1];
1746                         pktnb_stats[2] = pktnb_stats[1];
1747                         burst_stats[1] = nb_burst;
1748                         pktnb_stats[1] = nb_pkt;
1749                 } else if (nb_burst > burst_stats[2]) {
1750                         burst_stats[2] = nb_burst;
1751                         pktnb_stats[2] = nb_pkt;
1752                 }
1753         }
1754         if (total_burst == 0)
1755                 return;
1756
1757         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1758         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1759                 if (i == 3) {
1760                         printf("%d%% of other]\n", 100 - sburstp);
1761                         return;
1762                 }
1763
1764                 sburst += burst_stats[i];
1765                 if (sburst == total_burst) {
1766                         printf("%d%% of %d pkts]\n",
1767                                 100 - sburstp, (int) pktnb_stats[i]);
1768                         return;
1769                 }
1770
1771                 burst_percent[i] =
1772                         (double)burst_stats[i] / total_burst * 100;
1773                 printf("%d%% of %d pkts + ",
1774                         burst_percent[i], (int) pktnb_stats[i]);
1775                 sburstp += burst_percent[i];
1776         }
1777 }
1778
1779 static void
1780 fwd_stream_stats_display(streamid_t stream_id)
1781 {
1782         struct fwd_stream *fs;
1783         static const char *fwd_top_stats_border = "-------";
1784
1785         fs = fwd_streams[stream_id];
1786         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1787             (fs->fwd_dropped == 0))
1788                 return;
1789         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1790                "TX Port=%2d/Queue=%2d %s\n",
1791                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1792                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1793         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1794                " TX-dropped: %-14"PRIu64,
1795                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1796
1797         /* if checksum mode */
1798         if (cur_fwd_eng == &csum_fwd_engine) {
1799                 printf("  RX- bad IP checksum: %-14"PRIu64
1800                        "  Rx- bad L4 checksum: %-14"PRIu64
1801                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1802                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1803                         fs->rx_bad_outer_l4_csum);
1804                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1805                         fs->rx_bad_outer_ip_csum);
1806         } else {
1807                 printf("\n");
1808         }
1809
1810         if (record_burst_stats) {
1811                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1812                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1813         }
1814 }
1815
1816 void
1817 fwd_stats_display(void)
1818 {
1819         static const char *fwd_stats_border = "----------------------";
1820         static const char *acc_stats_border = "+++++++++++++++";
1821         struct {
1822                 struct fwd_stream *rx_stream;
1823                 struct fwd_stream *tx_stream;
1824                 uint64_t tx_dropped;
1825                 uint64_t rx_bad_ip_csum;
1826                 uint64_t rx_bad_l4_csum;
1827                 uint64_t rx_bad_outer_l4_csum;
1828                 uint64_t rx_bad_outer_ip_csum;
1829         } ports_stats[RTE_MAX_ETHPORTS];
1830         uint64_t total_rx_dropped = 0;
1831         uint64_t total_tx_dropped = 0;
1832         uint64_t total_rx_nombuf = 0;
1833         struct rte_eth_stats stats;
1834         uint64_t fwd_cycles = 0;
1835         uint64_t total_recv = 0;
1836         uint64_t total_xmit = 0;
1837         struct rte_port *port;
1838         streamid_t sm_id;
1839         portid_t pt_id;
1840         int i;
1841
1842         memset(ports_stats, 0, sizeof(ports_stats));
1843
1844         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1845                 struct fwd_stream *fs = fwd_streams[sm_id];
1846
1847                 if (cur_fwd_config.nb_fwd_streams >
1848                     cur_fwd_config.nb_fwd_ports) {
1849                         fwd_stream_stats_display(sm_id);
1850                 } else {
1851                         ports_stats[fs->tx_port].tx_stream = fs;
1852                         ports_stats[fs->rx_port].rx_stream = fs;
1853                 }
1854
1855                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1856
1857                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1858                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1859                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1860                                 fs->rx_bad_outer_l4_csum;
1861                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1862                                 fs->rx_bad_outer_ip_csum;
1863
1864                 if (record_core_cycles)
1865                         fwd_cycles += fs->core_cycles;
1866         }
1867         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1868                 pt_id = fwd_ports_ids[i];
1869                 port = &ports[pt_id];
1870
1871                 rte_eth_stats_get(pt_id, &stats);
1872                 stats.ipackets -= port->stats.ipackets;
1873                 stats.opackets -= port->stats.opackets;
1874                 stats.ibytes -= port->stats.ibytes;
1875                 stats.obytes -= port->stats.obytes;
1876                 stats.imissed -= port->stats.imissed;
1877                 stats.oerrors -= port->stats.oerrors;
1878                 stats.rx_nombuf -= port->stats.rx_nombuf;
1879
1880                 total_recv += stats.ipackets;
1881                 total_xmit += stats.opackets;
1882                 total_rx_dropped += stats.imissed;
1883                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1884                 total_tx_dropped += stats.oerrors;
1885                 total_rx_nombuf  += stats.rx_nombuf;
1886
1887                 printf("\n  %s Forward statistics for port %-2d %s\n",
1888                        fwd_stats_border, pt_id, fwd_stats_border);
1889
1890                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1891                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1892                        stats.ipackets + stats.imissed);
1893
1894                 if (cur_fwd_eng == &csum_fwd_engine) {
1895                         printf("  Bad-ipcsum: %-14"PRIu64
1896                                " Bad-l4csum: %-14"PRIu64
1897                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1898                                ports_stats[pt_id].rx_bad_ip_csum,
1899                                ports_stats[pt_id].rx_bad_l4_csum,
1900                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1901                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1902                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1903                 }
1904                 if (stats.ierrors + stats.rx_nombuf > 0) {
1905                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1906                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1907                 }
1908
1909                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1910                        "TX-total: %-"PRIu64"\n",
1911                        stats.opackets, ports_stats[pt_id].tx_dropped,
1912                        stats.opackets + ports_stats[pt_id].tx_dropped);
1913
1914                 if (record_burst_stats) {
1915                         if (ports_stats[pt_id].rx_stream)
1916                                 pkt_burst_stats_display("RX",
1917                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1918                         if (ports_stats[pt_id].tx_stream)
1919                                 pkt_burst_stats_display("TX",
1920                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1921                 }
1922
1923                 printf("  %s--------------------------------%s\n",
1924                        fwd_stats_border, fwd_stats_border);
1925         }
1926
1927         printf("\n  %s Accumulated forward statistics for all ports"
1928                "%s\n",
1929                acc_stats_border, acc_stats_border);
1930         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1931                "%-"PRIu64"\n"
1932                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1933                "%-"PRIu64"\n",
1934                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1935                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1936         if (total_rx_nombuf > 0)
1937                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1938         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1939                "%s\n",
1940                acc_stats_border, acc_stats_border);
1941         if (record_core_cycles) {
1942 #define CYC_PER_MHZ 1E6
1943                 if (total_recv > 0 || total_xmit > 0) {
1944                         uint64_t total_pkts = 0;
1945                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1946                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1947                                 total_pkts = total_xmit;
1948                         else
1949                                 total_pkts = total_recv;
1950
1951                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1952                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1953                                " MHz Clock\n",
1954                                (double) fwd_cycles / total_pkts,
1955                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1956                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1957                 }
1958         }
1959 }
1960
1961 void
1962 fwd_stats_reset(void)
1963 {
1964         streamid_t sm_id;
1965         portid_t pt_id;
1966         int i;
1967
1968         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1969                 pt_id = fwd_ports_ids[i];
1970                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1971         }
1972         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1973                 struct fwd_stream *fs = fwd_streams[sm_id];
1974
1975                 fs->rx_packets = 0;
1976                 fs->tx_packets = 0;
1977                 fs->fwd_dropped = 0;
1978                 fs->rx_bad_ip_csum = 0;
1979                 fs->rx_bad_l4_csum = 0;
1980                 fs->rx_bad_outer_l4_csum = 0;
1981                 fs->rx_bad_outer_ip_csum = 0;
1982
1983                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1984                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1985                 fs->core_cycles = 0;
1986         }
1987 }
1988
1989 static void
1990 flush_fwd_rx_queues(void)
1991 {
1992         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1993         portid_t  rxp;
1994         portid_t port_id;
1995         queueid_t rxq;
1996         uint16_t  nb_rx;
1997         uint16_t  i;
1998         uint8_t   j;
1999         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2000         uint64_t timer_period;
2001
2002         /* convert to number of cycles */
2003         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2004
2005         for (j = 0; j < 2; j++) {
2006                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2007                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2008                                 port_id = fwd_ports_ids[rxp];
2009                                 /**
2010                                 * testpmd can stuck in the below do while loop
2011                                 * if rte_eth_rx_burst() always returns nonzero
2012                                 * packets. So timer is added to exit this loop
2013                                 * after 1sec timer expiry.
2014                                 */
2015                                 prev_tsc = rte_rdtsc();
2016                                 do {
2017                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2018                                                 pkts_burst, MAX_PKT_BURST);
2019                                         for (i = 0; i < nb_rx; i++)
2020                                                 rte_pktmbuf_free(pkts_burst[i]);
2021
2022                                         cur_tsc = rte_rdtsc();
2023                                         diff_tsc = cur_tsc - prev_tsc;
2024                                         timer_tsc += diff_tsc;
2025                                 } while ((nb_rx > 0) &&
2026                                         (timer_tsc < timer_period));
2027                                 timer_tsc = 0;
2028                         }
2029                 }
2030                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2031         }
2032 }
2033
2034 static void
2035 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2036 {
2037         struct fwd_stream **fsm;
2038         streamid_t nb_fs;
2039         streamid_t sm_id;
2040 #ifdef RTE_LIB_BITRATESTATS
2041         uint64_t tics_per_1sec;
2042         uint64_t tics_datum;
2043         uint64_t tics_current;
2044         uint16_t i, cnt_ports;
2045
2046         cnt_ports = nb_ports;
2047         tics_datum = rte_rdtsc();
2048         tics_per_1sec = rte_get_timer_hz();
2049 #endif
2050         fsm = &fwd_streams[fc->stream_idx];
2051         nb_fs = fc->stream_nb;
2052         do {
2053                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2054                         (*pkt_fwd)(fsm[sm_id]);
2055 #ifdef RTE_LIB_BITRATESTATS
2056                 if (bitrate_enabled != 0 &&
2057                                 bitrate_lcore_id == rte_lcore_id()) {
2058                         tics_current = rte_rdtsc();
2059                         if (tics_current - tics_datum >= tics_per_1sec) {
2060                                 /* Periodic bitrate calculation */
2061                                 for (i = 0; i < cnt_ports; i++)
2062                                         rte_stats_bitrate_calc(bitrate_data,
2063                                                 ports_ids[i]);
2064                                 tics_datum = tics_current;
2065                         }
2066                 }
2067 #endif
2068 #ifdef RTE_LIB_LATENCYSTATS
2069                 if (latencystats_enabled != 0 &&
2070                                 latencystats_lcore_id == rte_lcore_id())
2071                         rte_latencystats_update();
2072 #endif
2073
2074         } while (! fc->stopped);
2075 }
2076
2077 static int
2078 start_pkt_forward_on_core(void *fwd_arg)
2079 {
2080         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2081                              cur_fwd_config.fwd_eng->packet_fwd);
2082         return 0;
2083 }
2084
2085 /*
2086  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2087  * Used to start communication flows in network loopback test configurations.
2088  */
2089 static int
2090 run_one_txonly_burst_on_core(void *fwd_arg)
2091 {
2092         struct fwd_lcore *fwd_lc;
2093         struct fwd_lcore tmp_lcore;
2094
2095         fwd_lc = (struct fwd_lcore *) fwd_arg;
2096         tmp_lcore = *fwd_lc;
2097         tmp_lcore.stopped = 1;
2098         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2099         return 0;
2100 }
2101
2102 /*
2103  * Launch packet forwarding:
2104  *     - Setup per-port forwarding context.
2105  *     - launch logical cores with their forwarding configuration.
2106  */
2107 static void
2108 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2109 {
2110         port_fwd_begin_t port_fwd_begin;
2111         unsigned int i;
2112         unsigned int lc_id;
2113         int diag;
2114
2115         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2116         if (port_fwd_begin != NULL) {
2117                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2118                         (*port_fwd_begin)(fwd_ports_ids[i]);
2119         }
2120         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2121                 lc_id = fwd_lcores_cpuids[i];
2122                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2123                         fwd_lcores[i]->stopped = 0;
2124                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2125                                                      fwd_lcores[i], lc_id);
2126                         if (diag != 0)
2127                                 printf("launch lcore %u failed - diag=%d\n",
2128                                        lc_id, diag);
2129                 }
2130         }
2131 }
2132
2133 /*
2134  * Launch packet forwarding configuration.
2135  */
2136 void
2137 start_packet_forwarding(int with_tx_first)
2138 {
2139         port_fwd_begin_t port_fwd_begin;
2140         port_fwd_end_t  port_fwd_end;
2141         struct rte_port *port;
2142         unsigned int i;
2143         portid_t   pt_id;
2144
2145         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2146                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2147
2148         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2149                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2150
2151         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2152                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2153                 (!nb_rxq || !nb_txq))
2154                 rte_exit(EXIT_FAILURE,
2155                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2156                         cur_fwd_eng->fwd_mode_name);
2157
2158         if (all_ports_started() == 0) {
2159                 printf("Not all ports were started\n");
2160                 return;
2161         }
2162         if (test_done == 0) {
2163                 printf("Packet forwarding already started\n");
2164                 return;
2165         }
2166
2167         if (dcb_config) {
2168                 for (i = 0; i < nb_fwd_ports; i++) {
2169                         pt_id = fwd_ports_ids[i];
2170                         port = &ports[pt_id];
2171                         if (!port->dcb_flag) {
2172                                 printf("In DCB mode, all forwarding ports must "
2173                                        "be configured in this mode.\n");
2174                                 return;
2175                         }
2176                 }
2177                 if (nb_fwd_lcores == 1) {
2178                         printf("In DCB mode,the nb forwarding cores "
2179                                "should be larger than 1.\n");
2180                         return;
2181                 }
2182         }
2183         test_done = 0;
2184
2185         fwd_config_setup();
2186
2187         if(!no_flush_rx)
2188                 flush_fwd_rx_queues();
2189
2190         pkt_fwd_config_display(&cur_fwd_config);
2191         rxtx_config_display();
2192
2193         fwd_stats_reset();
2194         if (with_tx_first) {
2195                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2196                 if (port_fwd_begin != NULL) {
2197                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2198                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2199                 }
2200                 while (with_tx_first--) {
2201                         launch_packet_forwarding(
2202                                         run_one_txonly_burst_on_core);
2203                         rte_eal_mp_wait_lcore();
2204                 }
2205                 port_fwd_end = tx_only_engine.port_fwd_end;
2206                 if (port_fwd_end != NULL) {
2207                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2208                                 (*port_fwd_end)(fwd_ports_ids[i]);
2209                 }
2210         }
2211         launch_packet_forwarding(start_pkt_forward_on_core);
2212 }
2213
2214 void
2215 stop_packet_forwarding(void)
2216 {
2217         port_fwd_end_t port_fwd_end;
2218         lcoreid_t lc_id;
2219         portid_t pt_id;
2220         int i;
2221
2222         if (test_done) {
2223                 printf("Packet forwarding not started\n");
2224                 return;
2225         }
2226         printf("Telling cores to stop...");
2227         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2228                 fwd_lcores[lc_id]->stopped = 1;
2229         printf("\nWaiting for lcores to finish...\n");
2230         rte_eal_mp_wait_lcore();
2231         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2232         if (port_fwd_end != NULL) {
2233                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2234                         pt_id = fwd_ports_ids[i];
2235                         (*port_fwd_end)(pt_id);
2236                 }
2237         }
2238
2239         fwd_stats_display();
2240
2241         printf("\nDone.\n");
2242         test_done = 1;
2243 }
2244
2245 void
2246 dev_set_link_up(portid_t pid)
2247 {
2248         if (rte_eth_dev_set_link_up(pid) < 0)
2249                 printf("\nSet link up fail.\n");
2250 }
2251
2252 void
2253 dev_set_link_down(portid_t pid)
2254 {
2255         if (rte_eth_dev_set_link_down(pid) < 0)
2256                 printf("\nSet link down fail.\n");
2257 }
2258
2259 static int
2260 all_ports_started(void)
2261 {
2262         portid_t pi;
2263         struct rte_port *port;
2264
2265         RTE_ETH_FOREACH_DEV(pi) {
2266                 port = &ports[pi];
2267                 /* Check if there is a port which is not started */
2268                 if ((port->port_status != RTE_PORT_STARTED) &&
2269                         (port->slave_flag == 0))
2270                         return 0;
2271         }
2272
2273         /* No port is not started */
2274         return 1;
2275 }
2276
2277 int
2278 port_is_stopped(portid_t port_id)
2279 {
2280         struct rte_port *port = &ports[port_id];
2281
2282         if ((port->port_status != RTE_PORT_STOPPED) &&
2283             (port->slave_flag == 0))
2284                 return 0;
2285         return 1;
2286 }
2287
2288 int
2289 all_ports_stopped(void)
2290 {
2291         portid_t pi;
2292
2293         RTE_ETH_FOREACH_DEV(pi) {
2294                 if (!port_is_stopped(pi))
2295                         return 0;
2296         }
2297
2298         return 1;
2299 }
2300
2301 int
2302 port_is_started(portid_t port_id)
2303 {
2304         if (port_id_is_invalid(port_id, ENABLED_WARN))
2305                 return 0;
2306
2307         if (ports[port_id].port_status != RTE_PORT_STARTED)
2308                 return 0;
2309
2310         return 1;
2311 }
2312
2313 /* Configure the Rx and Tx hairpin queues for the selected port. */
2314 static int
2315 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2316 {
2317         queueid_t qi;
2318         struct rte_eth_hairpin_conf hairpin_conf = {
2319                 .peer_count = 1,
2320         };
2321         int i;
2322         int diag;
2323         struct rte_port *port = &ports[pi];
2324         uint16_t peer_rx_port = pi;
2325         uint16_t peer_tx_port = pi;
2326         uint32_t manual = 1;
2327         uint32_t tx_exp = hairpin_mode & 0x10;
2328
2329         if (!(hairpin_mode & 0xf)) {
2330                 peer_rx_port = pi;
2331                 peer_tx_port = pi;
2332                 manual = 0;
2333         } else if (hairpin_mode & 0x1) {
2334                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2335                                                        RTE_ETH_DEV_NO_OWNER);
2336                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2337                         peer_tx_port = rte_eth_find_next_owned_by(0,
2338                                                 RTE_ETH_DEV_NO_OWNER);
2339                 if (p_pi != RTE_MAX_ETHPORTS) {
2340                         peer_rx_port = p_pi;
2341                 } else {
2342                         uint16_t next_pi;
2343
2344                         /* Last port will be the peer RX port of the first. */
2345                         RTE_ETH_FOREACH_DEV(next_pi)
2346                                 peer_rx_port = next_pi;
2347                 }
2348                 manual = 1;
2349         } else if (hairpin_mode & 0x2) {
2350                 if (cnt_pi & 0x1) {
2351                         peer_rx_port = p_pi;
2352                 } else {
2353                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2354                                                 RTE_ETH_DEV_NO_OWNER);
2355                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2356                                 peer_rx_port = pi;
2357                 }
2358                 peer_tx_port = peer_rx_port;
2359                 manual = 1;
2360         }
2361
2362         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2363                 hairpin_conf.peers[0].port = peer_rx_port;
2364                 hairpin_conf.peers[0].queue = i + nb_rxq;
2365                 hairpin_conf.manual_bind = !!manual;
2366                 hairpin_conf.tx_explicit = !!tx_exp;
2367                 diag = rte_eth_tx_hairpin_queue_setup
2368                         (pi, qi, nb_txd, &hairpin_conf);
2369                 i++;
2370                 if (diag == 0)
2371                         continue;
2372
2373                 /* Fail to setup rx queue, return */
2374                 if (rte_atomic16_cmpset(&(port->port_status),
2375                                         RTE_PORT_HANDLING,
2376                                         RTE_PORT_STOPPED) == 0)
2377                         printf("Port %d can not be set back "
2378                                         "to stopped\n", pi);
2379                 printf("Fail to configure port %d hairpin "
2380                                 "queues\n", pi);
2381                 /* try to reconfigure queues next time */
2382                 port->need_reconfig_queues = 1;
2383                 return -1;
2384         }
2385         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2386                 hairpin_conf.peers[0].port = peer_tx_port;
2387                 hairpin_conf.peers[0].queue = i + nb_txq;
2388                 hairpin_conf.manual_bind = !!manual;
2389                 hairpin_conf.tx_explicit = !!tx_exp;
2390                 diag = rte_eth_rx_hairpin_queue_setup
2391                         (pi, qi, nb_rxd, &hairpin_conf);
2392                 i++;
2393                 if (diag == 0)
2394                         continue;
2395
2396                 /* Fail to setup rx queue, return */
2397                 if (rte_atomic16_cmpset(&(port->port_status),
2398                                         RTE_PORT_HANDLING,
2399                                         RTE_PORT_STOPPED) == 0)
2400                         printf("Port %d can not be set back "
2401                                         "to stopped\n", pi);
2402                 printf("Fail to configure port %d hairpin "
2403                                 "queues\n", pi);
2404                 /* try to reconfigure queues next time */
2405                 port->need_reconfig_queues = 1;
2406                 return -1;
2407         }
2408         return 0;
2409 }
2410
2411 /* Configure the Rx with optional split. */
2412 int
2413 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2414                uint16_t nb_rx_desc, unsigned int socket_id,
2415                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2416 {
2417         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2418         unsigned int i, mp_n;
2419         int ret;
2420
2421         if (rx_pkt_nb_segs <= 1 ||
2422             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2423                 rx_conf->rx_seg = NULL;
2424                 rx_conf->rx_nseg = 0;
2425                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2426                                              nb_rx_desc, socket_id,
2427                                              rx_conf, mp);
2428                 return ret;
2429         }
2430         for (i = 0; i < rx_pkt_nb_segs; i++) {
2431                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2432                 struct rte_mempool *mpx;
2433                 /*
2434                  * Use last valid pool for the segments with number
2435                  * exceeding the pool index.
2436                  */
2437                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2438                 mpx = mbuf_pool_find(socket_id, mp_n);
2439                 /* Handle zero as mbuf data buffer size. */
2440                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2441                                    rx_pkt_seg_lengths[i] :
2442                                    mbuf_data_size[mp_n];
2443                 rx_seg->offset = i < rx_pkt_nb_offs ?
2444                                    rx_pkt_seg_offsets[i] : 0;
2445                 rx_seg->mp = mpx ? mpx : mp;
2446         }
2447         rx_conf->rx_nseg = rx_pkt_nb_segs;
2448         rx_conf->rx_seg = rx_useg;
2449         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2450                                     socket_id, rx_conf, NULL);
2451         rx_conf->rx_seg = NULL;
2452         rx_conf->rx_nseg = 0;
2453         return ret;
2454 }
2455
2456 int
2457 start_port(portid_t pid)
2458 {
2459         int diag, need_check_link_status = -1;
2460         portid_t pi;
2461         portid_t p_pi = RTE_MAX_ETHPORTS;
2462         portid_t pl[RTE_MAX_ETHPORTS];
2463         portid_t peer_pl[RTE_MAX_ETHPORTS];
2464         uint16_t cnt_pi = 0;
2465         uint16_t cfg_pi = 0;
2466         int peer_pi;
2467         queueid_t qi;
2468         struct rte_port *port;
2469         struct rte_ether_addr mac_addr;
2470         struct rte_eth_hairpin_cap cap;
2471
2472         if (port_id_is_invalid(pid, ENABLED_WARN))
2473                 return 0;
2474
2475         RTE_ETH_FOREACH_DEV(pi) {
2476                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2477                         continue;
2478
2479                 need_check_link_status = 0;
2480                 port = &ports[pi];
2481                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2482                                                  RTE_PORT_HANDLING) == 0) {
2483                         printf("Port %d is now not stopped\n", pi);
2484                         continue;
2485                 }
2486
2487                 if (port->need_reconfig > 0) {
2488                         port->need_reconfig = 0;
2489
2490                         if (flow_isolate_all) {
2491                                 int ret = port_flow_isolate(pi, 1);
2492                                 if (ret) {
2493                                         printf("Failed to apply isolated"
2494                                                " mode on port %d\n", pi);
2495                                         return -1;
2496                                 }
2497                         }
2498                         configure_rxtx_dump_callbacks(0);
2499                         printf("Configuring Port %d (socket %u)\n", pi,
2500                                         port->socket_id);
2501                         if (nb_hairpinq > 0 &&
2502                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2503                                 printf("Port %d doesn't support hairpin "
2504                                        "queues\n", pi);
2505                                 return -1;
2506                         }
2507                         /* configure port */
2508                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2509                                                      nb_txq + nb_hairpinq,
2510                                                      &(port->dev_conf));
2511                         if (diag != 0) {
2512                                 if (rte_atomic16_cmpset(&(port->port_status),
2513                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2514                                         printf("Port %d can not be set back "
2515                                                         "to stopped\n", pi);
2516                                 printf("Fail to configure port %d\n", pi);
2517                                 /* try to reconfigure port next time */
2518                                 port->need_reconfig = 1;
2519                                 return -1;
2520                         }
2521                 }
2522                 if (port->need_reconfig_queues > 0) {
2523                         port->need_reconfig_queues = 0;
2524                         /* setup tx queues */
2525                         for (qi = 0; qi < nb_txq; qi++) {
2526                                 if ((numa_support) &&
2527                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2528                                         diag = rte_eth_tx_queue_setup(pi, qi,
2529                                                 port->nb_tx_desc[qi],
2530                                                 txring_numa[pi],
2531                                                 &(port->tx_conf[qi]));
2532                                 else
2533                                         diag = rte_eth_tx_queue_setup(pi, qi,
2534                                                 port->nb_tx_desc[qi],
2535                                                 port->socket_id,
2536                                                 &(port->tx_conf[qi]));
2537
2538                                 if (diag == 0)
2539                                         continue;
2540
2541                                 /* Fail to setup tx queue, return */
2542                                 if (rte_atomic16_cmpset(&(port->port_status),
2543                                                         RTE_PORT_HANDLING,
2544                                                         RTE_PORT_STOPPED) == 0)
2545                                         printf("Port %d can not be set back "
2546                                                         "to stopped\n", pi);
2547                                 printf("Fail to configure port %d tx queues\n",
2548                                        pi);
2549                                 /* try to reconfigure queues next time */
2550                                 port->need_reconfig_queues = 1;
2551                                 return -1;
2552                         }
2553                         for (qi = 0; qi < nb_rxq; qi++) {
2554                                 /* setup rx queues */
2555                                 if ((numa_support) &&
2556                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2557                                         struct rte_mempool * mp =
2558                                                 mbuf_pool_find
2559                                                         (rxring_numa[pi], 0);
2560                                         if (mp == NULL) {
2561                                                 printf("Failed to setup RX queue:"
2562                                                         "No mempool allocation"
2563                                                         " on the socket %d\n",
2564                                                         rxring_numa[pi]);
2565                                                 return -1;
2566                                         }
2567
2568                                         diag = rx_queue_setup(pi, qi,
2569                                              port->nb_rx_desc[qi],
2570                                              rxring_numa[pi],
2571                                              &(port->rx_conf[qi]),
2572                                              mp);
2573                                 } else {
2574                                         struct rte_mempool *mp =
2575                                                 mbuf_pool_find
2576                                                         (port->socket_id, 0);
2577                                         if (mp == NULL) {
2578                                                 printf("Failed to setup RX queue:"
2579                                                         "No mempool allocation"
2580                                                         " on the socket %d\n",
2581                                                         port->socket_id);
2582                                                 return -1;
2583                                         }
2584                                         diag = rx_queue_setup(pi, qi,
2585                                              port->nb_rx_desc[qi],
2586                                              port->socket_id,
2587                                              &(port->rx_conf[qi]),
2588                                              mp);
2589                                 }
2590                                 if (diag == 0)
2591                                         continue;
2592
2593                                 /* Fail to setup rx queue, return */
2594                                 if (rte_atomic16_cmpset(&(port->port_status),
2595                                                         RTE_PORT_HANDLING,
2596                                                         RTE_PORT_STOPPED) == 0)
2597                                         printf("Port %d can not be set back "
2598                                                         "to stopped\n", pi);
2599                                 printf("Fail to configure port %d rx queues\n",
2600                                        pi);
2601                                 /* try to reconfigure queues next time */
2602                                 port->need_reconfig_queues = 1;
2603                                 return -1;
2604                         }
2605                         /* setup hairpin queues */
2606                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2607                                 return -1;
2608                 }
2609                 configure_rxtx_dump_callbacks(verbose_level);
2610                 if (clear_ptypes) {
2611                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2612                                         NULL, 0);
2613                         if (diag < 0)
2614                                 printf(
2615                                 "Port %d: Failed to disable Ptype parsing\n",
2616                                 pi);
2617                 }
2618
2619                 p_pi = pi;
2620                 cnt_pi++;
2621
2622                 /* start port */
2623                 diag = rte_eth_dev_start(pi);
2624                 if (diag < 0) {
2625                         printf("Fail to start port %d: %s\n", pi,
2626                                rte_strerror(-diag));
2627
2628                         /* Fail to setup rx queue, return */
2629                         if (rte_atomic16_cmpset(&(port->port_status),
2630                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2631                                 printf("Port %d can not be set back to "
2632                                                         "stopped\n", pi);
2633                         continue;
2634                 }
2635
2636                 if (rte_atomic16_cmpset(&(port->port_status),
2637                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2638                         printf("Port %d can not be set into started\n", pi);
2639
2640                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2641                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2642                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2643                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2644                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2645
2646                 /* at least one port started, need checking link status */
2647                 need_check_link_status = 1;
2648
2649                 pl[cfg_pi++] = pi;
2650         }
2651
2652         if (need_check_link_status == 1 && !no_link_check)
2653                 check_all_ports_link_status(RTE_PORT_ALL);
2654         else if (need_check_link_status == 0)
2655                 printf("Please stop the ports first\n");
2656
2657         if (hairpin_mode & 0xf) {
2658                 uint16_t i;
2659                 int j;
2660
2661                 /* bind all started hairpin ports */
2662                 for (i = 0; i < cfg_pi; i++) {
2663                         pi = pl[i];
2664                         /* bind current Tx to all peer Rx */
2665                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2666                                                         RTE_MAX_ETHPORTS, 1);
2667                         if (peer_pi < 0)
2668                                 return peer_pi;
2669                         for (j = 0; j < peer_pi; j++) {
2670                                 if (!port_is_started(peer_pl[j]))
2671                                         continue;
2672                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2673                                 if (diag < 0) {
2674                                         printf("Error during binding hairpin"
2675                                                " Tx port %u to %u: %s\n",
2676                                                pi, peer_pl[j],
2677                                                rte_strerror(-diag));
2678                                         return -1;
2679                                 }
2680                         }
2681                         /* bind all peer Tx to current Rx */
2682                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2683                                                         RTE_MAX_ETHPORTS, 0);
2684                         if (peer_pi < 0)
2685                                 return peer_pi;
2686                         for (j = 0; j < peer_pi; j++) {
2687                                 if (!port_is_started(peer_pl[j]))
2688                                         continue;
2689                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2690                                 if (diag < 0) {
2691                                         printf("Error during binding hairpin"
2692                                                " Tx port %u to %u: %s\n",
2693                                                peer_pl[j], pi,
2694                                                rte_strerror(-diag));
2695                                         return -1;
2696                                 }
2697                         }
2698                 }
2699         }
2700
2701         printf("Done\n");
2702         return 0;
2703 }
2704
2705 void
2706 stop_port(portid_t pid)
2707 {
2708         portid_t pi;
2709         struct rte_port *port;
2710         int need_check_link_status = 0;
2711         portid_t peer_pl[RTE_MAX_ETHPORTS];
2712         int peer_pi;
2713
2714         if (port_id_is_invalid(pid, ENABLED_WARN))
2715                 return;
2716
2717         printf("Stopping ports...\n");
2718
2719         RTE_ETH_FOREACH_DEV(pi) {
2720                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2721                         continue;
2722
2723                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2724                         printf("Please remove port %d from forwarding configuration.\n", pi);
2725                         continue;
2726                 }
2727
2728                 if (port_is_bonding_slave(pi)) {
2729                         printf("Please remove port %d from bonded device.\n", pi);
2730                         continue;
2731                 }
2732
2733                 port = &ports[pi];
2734                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2735                                                 RTE_PORT_HANDLING) == 0)
2736                         continue;
2737
2738                 if (hairpin_mode & 0xf) {
2739                         int j;
2740
2741                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2742                         /* unbind all peer Tx from current Rx */
2743                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2744                                                         RTE_MAX_ETHPORTS, 0);
2745                         if (peer_pi < 0)
2746                                 continue;
2747                         for (j = 0; j < peer_pi; j++) {
2748                                 if (!port_is_started(peer_pl[j]))
2749                                         continue;
2750                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2751                         }
2752                 }
2753
2754                 if (port->flow_list)
2755                         port_flow_flush(pi);
2756
2757                 if (rte_eth_dev_stop(pi) != 0)
2758                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2759                                 pi);
2760
2761                 if (rte_atomic16_cmpset(&(port->port_status),
2762                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2763                         printf("Port %d can not be set into stopped\n", pi);
2764                 need_check_link_status = 1;
2765         }
2766         if (need_check_link_status && !no_link_check)
2767                 check_all_ports_link_status(RTE_PORT_ALL);
2768
2769         printf("Done\n");
2770 }
2771
2772 static void
2773 remove_invalid_ports_in(portid_t *array, portid_t *total)
2774 {
2775         portid_t i;
2776         portid_t new_total = 0;
2777
2778         for (i = 0; i < *total; i++)
2779                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2780                         array[new_total] = array[i];
2781                         new_total++;
2782                 }
2783         *total = new_total;
2784 }
2785
2786 static void
2787 remove_invalid_ports(void)
2788 {
2789         remove_invalid_ports_in(ports_ids, &nb_ports);
2790         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2791         nb_cfg_ports = nb_fwd_ports;
2792 }
2793
2794 void
2795 close_port(portid_t pid)
2796 {
2797         portid_t pi;
2798         struct rte_port *port;
2799
2800         if (port_id_is_invalid(pid, ENABLED_WARN))
2801                 return;
2802
2803         printf("Closing ports...\n");
2804
2805         RTE_ETH_FOREACH_DEV(pi) {
2806                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2807                         continue;
2808
2809                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2810                         printf("Please remove port %d from forwarding configuration.\n", pi);
2811                         continue;
2812                 }
2813
2814                 if (port_is_bonding_slave(pi)) {
2815                         printf("Please remove port %d from bonded device.\n", pi);
2816                         continue;
2817                 }
2818
2819                 port = &ports[pi];
2820                 if (rte_atomic16_cmpset(&(port->port_status),
2821                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2822                         printf("Port %d is already closed\n", pi);
2823                         continue;
2824                 }
2825
2826                 port_flow_flush(pi);
2827                 rte_eth_dev_close(pi);
2828         }
2829
2830         remove_invalid_ports();
2831         printf("Done\n");
2832 }
2833
2834 void
2835 reset_port(portid_t pid)
2836 {
2837         int diag;
2838         portid_t pi;
2839         struct rte_port *port;
2840
2841         if (port_id_is_invalid(pid, ENABLED_WARN))
2842                 return;
2843
2844         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2845                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2846                 printf("Can not reset port(s), please stop port(s) first.\n");
2847                 return;
2848         }
2849
2850         printf("Resetting ports...\n");
2851
2852         RTE_ETH_FOREACH_DEV(pi) {
2853                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2854                         continue;
2855
2856                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2857                         printf("Please remove port %d from forwarding "
2858                                "configuration.\n", pi);
2859                         continue;
2860                 }
2861
2862                 if (port_is_bonding_slave(pi)) {
2863                         printf("Please remove port %d from bonded device.\n",
2864                                pi);
2865                         continue;
2866                 }
2867
2868                 diag = rte_eth_dev_reset(pi);
2869                 if (diag == 0) {
2870                         port = &ports[pi];
2871                         port->need_reconfig = 1;
2872                         port->need_reconfig_queues = 1;
2873                 } else {
2874                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2875                 }
2876         }
2877
2878         printf("Done\n");
2879 }
2880
2881 void
2882 attach_port(char *identifier)
2883 {
2884         portid_t pi;
2885         struct rte_dev_iterator iterator;
2886
2887         printf("Attaching a new port...\n");
2888
2889         if (identifier == NULL) {
2890                 printf("Invalid parameters are specified\n");
2891                 return;
2892         }
2893
2894         if (rte_dev_probe(identifier) < 0) {
2895                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2896                 return;
2897         }
2898
2899         /* first attach mode: event */
2900         if (setup_on_probe_event) {
2901                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2902                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2903                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2904                                         ports[pi].need_setup != 0)
2905                                 setup_attached_port(pi);
2906                 return;
2907         }
2908
2909         /* second attach mode: iterator */
2910         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2911                 /* setup ports matching the devargs used for probing */
2912                 if (port_is_forwarding(pi))
2913                         continue; /* port was already attached before */
2914                 setup_attached_port(pi);
2915         }
2916 }
2917
2918 static void
2919 setup_attached_port(portid_t pi)
2920 {
2921         unsigned int socket_id;
2922         int ret;
2923
2924         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2925         /* if socket_id is invalid, set to the first available socket. */
2926         if (check_socket_id(socket_id) < 0)
2927                 socket_id = socket_ids[0];
2928         reconfig(pi, socket_id);
2929         ret = rte_eth_promiscuous_enable(pi);
2930         if (ret != 0)
2931                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2932                         pi, rte_strerror(-ret));
2933
2934         ports_ids[nb_ports++] = pi;
2935         fwd_ports_ids[nb_fwd_ports++] = pi;
2936         nb_cfg_ports = nb_fwd_ports;
2937         ports[pi].need_setup = 0;
2938         ports[pi].port_status = RTE_PORT_STOPPED;
2939
2940         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2941         printf("Done\n");
2942 }
2943
2944 static void
2945 detach_device(struct rte_device *dev)
2946 {
2947         portid_t sibling;
2948
2949         if (dev == NULL) {
2950                 printf("Device already removed\n");
2951                 return;
2952         }
2953
2954         printf("Removing a device...\n");
2955
2956         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2957                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2958                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2959                                 printf("Port %u not stopped\n", sibling);
2960                                 return;
2961                         }
2962                         port_flow_flush(sibling);
2963                 }
2964         }
2965
2966         if (rte_dev_remove(dev) < 0) {
2967                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2968                 return;
2969         }
2970         remove_invalid_ports();
2971
2972         printf("Device is detached\n");
2973         printf("Now total ports is %d\n", nb_ports);
2974         printf("Done\n");
2975         return;
2976 }
2977
2978 void
2979 detach_port_device(portid_t port_id)
2980 {
2981         if (port_id_is_invalid(port_id, ENABLED_WARN))
2982                 return;
2983
2984         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2985                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2986                         printf("Port not stopped\n");
2987                         return;
2988                 }
2989                 printf("Port was not closed\n");
2990         }
2991
2992         detach_device(rte_eth_devices[port_id].device);
2993 }
2994
2995 void
2996 detach_devargs(char *identifier)
2997 {
2998         struct rte_dev_iterator iterator;
2999         struct rte_devargs da;
3000         portid_t port_id;
3001
3002         printf("Removing a device...\n");
3003
3004         memset(&da, 0, sizeof(da));
3005         if (rte_devargs_parsef(&da, "%s", identifier)) {
3006                 printf("cannot parse identifier\n");
3007                 return;
3008         }
3009
3010         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3011                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3012                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3013                                 printf("Port %u not stopped\n", port_id);
3014                                 rte_eth_iterator_cleanup(&iterator);
3015                                 rte_devargs_reset(&da);
3016                                 return;
3017                         }
3018                         port_flow_flush(port_id);
3019                 }
3020         }
3021
3022         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3023                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3024                             da.name, da.bus->name);
3025                 rte_devargs_reset(&da);
3026                 return;
3027         }
3028
3029         remove_invalid_ports();
3030
3031         printf("Device %s is detached\n", identifier);
3032         printf("Now total ports is %d\n", nb_ports);
3033         printf("Done\n");
3034         rte_devargs_reset(&da);
3035 }
3036
3037 void
3038 pmd_test_exit(void)
3039 {
3040         portid_t pt_id;
3041         unsigned int i;
3042         int ret;
3043
3044         if (test_done == 0)
3045                 stop_packet_forwarding();
3046
3047         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3048                 if (mempools[i]) {
3049                         if (mp_alloc_type == MP_ALLOC_ANON)
3050                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3051                                                      NULL);
3052                 }
3053         }
3054         if (ports != NULL) {
3055                 no_link_check = 1;
3056                 RTE_ETH_FOREACH_DEV(pt_id) {
3057                         printf("\nStopping port %d...\n", pt_id);
3058                         fflush(stdout);
3059                         stop_port(pt_id);
3060                 }
3061                 RTE_ETH_FOREACH_DEV(pt_id) {
3062                         printf("\nShutting down port %d...\n", pt_id);
3063                         fflush(stdout);
3064                         close_port(pt_id);
3065                 }
3066         }
3067
3068         if (hot_plug) {
3069                 ret = rte_dev_event_monitor_stop();
3070                 if (ret) {
3071                         RTE_LOG(ERR, EAL,
3072                                 "fail to stop device event monitor.");
3073                         return;
3074                 }
3075
3076                 ret = rte_dev_event_callback_unregister(NULL,
3077                         dev_event_callback, NULL);
3078                 if (ret < 0) {
3079                         RTE_LOG(ERR, EAL,
3080                                 "fail to unregister device event callback.\n");
3081                         return;
3082                 }
3083
3084                 ret = rte_dev_hotplug_handle_disable();
3085                 if (ret) {
3086                         RTE_LOG(ERR, EAL,
3087                                 "fail to disable hotplug handling.\n");
3088                         return;
3089                 }
3090         }
3091         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3092                 if (mempools[i])
3093                         rte_mempool_free(mempools[i]);
3094         }
3095
3096         printf("\nBye...\n");
3097 }
3098
3099 typedef void (*cmd_func_t)(void);
3100 struct pmd_test_command {
3101         const char *cmd_name;
3102         cmd_func_t cmd_func;
3103 };
3104
3105 /* Check the link status of all ports in up to 9s, and print them finally */
3106 static void
3107 check_all_ports_link_status(uint32_t port_mask)
3108 {
3109 #define CHECK_INTERVAL 100 /* 100ms */
3110 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3111         portid_t portid;
3112         uint8_t count, all_ports_up, print_flag = 0;
3113         struct rte_eth_link link;
3114         int ret;
3115         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3116
3117         printf("Checking link statuses...\n");
3118         fflush(stdout);
3119         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3120                 all_ports_up = 1;
3121                 RTE_ETH_FOREACH_DEV(portid) {
3122                         if ((port_mask & (1 << portid)) == 0)
3123                                 continue;
3124                         memset(&link, 0, sizeof(link));
3125                         ret = rte_eth_link_get_nowait(portid, &link);
3126                         if (ret < 0) {
3127                                 all_ports_up = 0;
3128                                 if (print_flag == 1)
3129                                         printf("Port %u link get failed: %s\n",
3130                                                 portid, rte_strerror(-ret));
3131                                 continue;
3132                         }
3133                         /* print link status if flag set */
3134                         if (print_flag == 1) {
3135                                 rte_eth_link_to_str(link_status,
3136                                         sizeof(link_status), &link);
3137                                 printf("Port %d %s\n", portid, link_status);
3138                                 continue;
3139                         }
3140                         /* clear all_ports_up flag if any link down */
3141                         if (link.link_status == ETH_LINK_DOWN) {
3142                                 all_ports_up = 0;
3143                                 break;
3144                         }
3145                 }
3146                 /* after finally printing all link status, get out */
3147                 if (print_flag == 1)
3148                         break;
3149
3150                 if (all_ports_up == 0) {
3151                         fflush(stdout);
3152                         rte_delay_ms(CHECK_INTERVAL);
3153                 }
3154
3155                 /* set the print_flag if all ports up or timeout */
3156                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3157                         print_flag = 1;
3158                 }
3159
3160                 if (lsc_interrupt)
3161                         break;
3162         }
3163 }
3164
3165 static void
3166 rmv_port_callback(void *arg)
3167 {
3168         int need_to_start = 0;
3169         int org_no_link_check = no_link_check;
3170         portid_t port_id = (intptr_t)arg;
3171         struct rte_device *dev;
3172
3173         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3174
3175         if (!test_done && port_is_forwarding(port_id)) {
3176                 need_to_start = 1;
3177                 stop_packet_forwarding();
3178         }
3179         no_link_check = 1;
3180         stop_port(port_id);
3181         no_link_check = org_no_link_check;
3182
3183         /* Save rte_device pointer before closing ethdev port */
3184         dev = rte_eth_devices[port_id].device;
3185         close_port(port_id);
3186         detach_device(dev); /* might be already removed or have more ports */
3187
3188         if (need_to_start)
3189                 start_packet_forwarding(0);
3190 }
3191
3192 /* This function is used by the interrupt thread */
3193 static int
3194 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3195                   void *ret_param)
3196 {
3197         RTE_SET_USED(param);
3198         RTE_SET_USED(ret_param);
3199
3200         if (type >= RTE_ETH_EVENT_MAX) {
3201                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3202                         port_id, __func__, type);
3203                 fflush(stderr);
3204         } else if (event_print_mask & (UINT32_C(1) << type)) {
3205                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3206                         eth_event_desc[type]);
3207                 fflush(stdout);
3208         }
3209
3210         switch (type) {
3211         case RTE_ETH_EVENT_NEW:
3212                 ports[port_id].need_setup = 1;
3213                 ports[port_id].port_status = RTE_PORT_HANDLING;
3214                 break;
3215         case RTE_ETH_EVENT_INTR_RMV:
3216                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3217                         break;
3218                 if (rte_eal_alarm_set(100000,
3219                                 rmv_port_callback, (void *)(intptr_t)port_id))
3220                         fprintf(stderr, "Could not set up deferred device removal\n");
3221                 break;
3222         case RTE_ETH_EVENT_DESTROY:
3223                 ports[port_id].port_status = RTE_PORT_CLOSED;
3224                 printf("Port %u is closed\n", port_id);
3225                 break;
3226         default:
3227                 break;
3228         }
3229         return 0;
3230 }
3231
3232 static int
3233 register_eth_event_callback(void)
3234 {
3235         int ret;
3236         enum rte_eth_event_type event;
3237
3238         for (event = RTE_ETH_EVENT_UNKNOWN;
3239                         event < RTE_ETH_EVENT_MAX; event++) {
3240                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3241                                 event,
3242                                 eth_event_callback,
3243                                 NULL);
3244                 if (ret != 0) {
3245                         TESTPMD_LOG(ERR, "Failed to register callback for "
3246                                         "%s event\n", eth_event_desc[event]);
3247                         return -1;
3248                 }
3249         }
3250
3251         return 0;
3252 }
3253
3254 /* This function is used by the interrupt thread */
3255 static void
3256 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3257                              __rte_unused void *arg)
3258 {
3259         uint16_t port_id;
3260         int ret;
3261
3262         if (type >= RTE_DEV_EVENT_MAX) {
3263                 fprintf(stderr, "%s called upon invalid event %d\n",
3264                         __func__, type);
3265                 fflush(stderr);
3266         }
3267
3268         switch (type) {
3269         case RTE_DEV_EVENT_REMOVE:
3270                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3271                         device_name);
3272                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3273                 if (ret) {
3274                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3275                                 device_name);
3276                         return;
3277                 }
3278                 /*
3279                  * Because the user's callback is invoked in eal interrupt
3280                  * callback, the interrupt callback need to be finished before
3281                  * it can be unregistered when detaching device. So finish
3282                  * callback soon and use a deferred removal to detach device
3283                  * is need. It is a workaround, once the device detaching be
3284                  * moved into the eal in the future, the deferred removal could
3285                  * be deleted.
3286                  */
3287                 if (rte_eal_alarm_set(100000,
3288                                 rmv_port_callback, (void *)(intptr_t)port_id))
3289                         RTE_LOG(ERR, EAL,
3290                                 "Could not set up deferred device removal\n");
3291                 break;
3292         case RTE_DEV_EVENT_ADD:
3293                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3294                         device_name);
3295                 /* TODO: After finish kernel driver binding,
3296                  * begin to attach port.
3297                  */
3298                 break;
3299         default:
3300                 break;
3301         }
3302 }
3303
3304 static void
3305 rxtx_port_config(struct rte_port *port)
3306 {
3307         uint16_t qid;
3308         uint64_t offloads;
3309
3310         for (qid = 0; qid < nb_rxq; qid++) {
3311                 offloads = port->rx_conf[qid].offloads;
3312                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3313                 if (offloads != 0)
3314                         port->rx_conf[qid].offloads = offloads;
3315
3316                 /* Check if any Rx parameters have been passed */
3317                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3318                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3319
3320                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3321                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3322
3323                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3324                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3325
3326                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3327                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3328
3329                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3330                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3331
3332                 port->nb_rx_desc[qid] = nb_rxd;
3333         }
3334
3335         for (qid = 0; qid < nb_txq; qid++) {
3336                 offloads = port->tx_conf[qid].offloads;
3337                 port->tx_conf[qid] = port->dev_info.default_txconf;
3338                 if (offloads != 0)
3339                         port->tx_conf[qid].offloads = offloads;
3340
3341                 /* Check if any Tx parameters have been passed */
3342                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3343                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3344
3345                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3346                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3347
3348                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3349                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3350
3351                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3352                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3353
3354                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3355                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3356
3357                 port->nb_tx_desc[qid] = nb_txd;
3358         }
3359 }
3360
3361 /*
3362  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3363  * MTU is also aligned if JUMBO_FRAME offload is not set.
3364  *
3365  * port->dev_info should be set before calling this function.
3366  *
3367  * return 0 on success, negative on error
3368  */
3369 int
3370 update_jumbo_frame_offload(portid_t portid)
3371 {
3372         struct rte_port *port = &ports[portid];
3373         uint32_t eth_overhead;
3374         uint64_t rx_offloads;
3375         int ret;
3376         bool on;
3377
3378         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3379         if (port->dev_info.max_mtu != UINT16_MAX &&
3380             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3381                 eth_overhead = port->dev_info.max_rx_pktlen -
3382                                 port->dev_info.max_mtu;
3383         else
3384                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3385
3386         rx_offloads = port->dev_conf.rxmode.offloads;
3387
3388         /* Default config value is 0 to use PMD specific overhead */
3389         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3390                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3391
3392         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3393                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3394                 on = false;
3395         } else {
3396                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3397                         printf("Frame size (%u) is not supported by port %u\n",
3398                                 port->dev_conf.rxmode.max_rx_pkt_len,
3399                                 portid);
3400                         return -1;
3401                 }
3402                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3403                 on = true;
3404         }
3405
3406         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3407                 uint16_t qid;
3408
3409                 port->dev_conf.rxmode.offloads = rx_offloads;
3410
3411                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3412                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3413                         if (on)
3414                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3415                         else
3416                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3417                 }
3418         }
3419
3420         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3421          * if unset do it here
3422          */
3423         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3424                 ret = rte_eth_dev_set_mtu(portid,
3425                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3426                 if (ret)
3427                         printf("Failed to set MTU to %u for port %u\n",
3428                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3429                                 portid);
3430         }
3431
3432         return 0;
3433 }
3434
3435 void
3436 init_port_config(void)
3437 {
3438         portid_t pid;
3439         struct rte_port *port;
3440         int ret;
3441
3442         RTE_ETH_FOREACH_DEV(pid) {
3443                 port = &ports[pid];
3444                 port->dev_conf.fdir_conf = fdir_conf;
3445
3446                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3447                 if (ret != 0)
3448                         return;
3449
3450                 if (nb_rxq > 1) {
3451                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3452                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3453                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3454                 } else {
3455                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3456                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3457                 }
3458
3459                 if (port->dcb_flag == 0) {
3460                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3461                                 port->dev_conf.rxmode.mq_mode =
3462                                         (enum rte_eth_rx_mq_mode)
3463                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3464                         else
3465                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3466                 }
3467
3468                 rxtx_port_config(port);
3469
3470                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3471                 if (ret != 0)
3472                         return;
3473
3474 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3475                 rte_pmd_ixgbe_bypass_init(pid);
3476 #endif
3477
3478                 if (lsc_interrupt &&
3479                     (rte_eth_devices[pid].data->dev_flags &
3480                      RTE_ETH_DEV_INTR_LSC))
3481                         port->dev_conf.intr_conf.lsc = 1;
3482                 if (rmv_interrupt &&
3483                     (rte_eth_devices[pid].data->dev_flags &
3484                      RTE_ETH_DEV_INTR_RMV))
3485                         port->dev_conf.intr_conf.rmv = 1;
3486         }
3487 }
3488
3489 void set_port_slave_flag(portid_t slave_pid)
3490 {
3491         struct rte_port *port;
3492
3493         port = &ports[slave_pid];
3494         port->slave_flag = 1;
3495 }
3496
3497 void clear_port_slave_flag(portid_t slave_pid)
3498 {
3499         struct rte_port *port;
3500
3501         port = &ports[slave_pid];
3502         port->slave_flag = 0;
3503 }
3504
3505 uint8_t port_is_bonding_slave(portid_t slave_pid)
3506 {
3507         struct rte_port *port;
3508
3509         port = &ports[slave_pid];
3510         if ((rte_eth_devices[slave_pid].data->dev_flags &
3511             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3512                 return 1;
3513         return 0;
3514 }
3515
3516 const uint16_t vlan_tags[] = {
3517                 0,  1,  2,  3,  4,  5,  6,  7,
3518                 8,  9, 10, 11,  12, 13, 14, 15,
3519                 16, 17, 18, 19, 20, 21, 22, 23,
3520                 24, 25, 26, 27, 28, 29, 30, 31
3521 };
3522
3523 static  int
3524 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3525                  enum dcb_mode_enable dcb_mode,
3526                  enum rte_eth_nb_tcs num_tcs,
3527                  uint8_t pfc_en)
3528 {
3529         uint8_t i;
3530         int32_t rc;
3531         struct rte_eth_rss_conf rss_conf;
3532
3533         /*
3534          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3535          * given above, and the number of traffic classes available for use.
3536          */
3537         if (dcb_mode == DCB_VT_ENABLED) {
3538                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3539                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3540                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3541                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3542
3543                 /* VMDQ+DCB RX and TX configurations */
3544                 vmdq_rx_conf->enable_default_pool = 0;
3545                 vmdq_rx_conf->default_pool = 0;
3546                 vmdq_rx_conf->nb_queue_pools =
3547                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3548                 vmdq_tx_conf->nb_queue_pools =
3549                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3550
3551                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3552                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3553                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3554                         vmdq_rx_conf->pool_map[i].pools =
3555                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3556                 }
3557                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3558                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3559                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3560                 }
3561
3562                 /* set DCB mode of RX and TX of multiple queues */
3563                 eth_conf->rxmode.mq_mode =
3564                                 (enum rte_eth_rx_mq_mode)
3565                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3566                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3567         } else {
3568                 struct rte_eth_dcb_rx_conf *rx_conf =
3569                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3570                 struct rte_eth_dcb_tx_conf *tx_conf =
3571                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3572
3573                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3574
3575                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3576                 if (rc != 0)
3577                         return rc;
3578
3579                 rx_conf->nb_tcs = num_tcs;
3580                 tx_conf->nb_tcs = num_tcs;
3581
3582                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3583                         rx_conf->dcb_tc[i] = i % num_tcs;
3584                         tx_conf->dcb_tc[i] = i % num_tcs;
3585                 }
3586
3587                 eth_conf->rxmode.mq_mode =
3588                                 (enum rte_eth_rx_mq_mode)
3589                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3590                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3591                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3592         }
3593
3594         if (pfc_en)
3595                 eth_conf->dcb_capability_en =
3596                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3597         else
3598                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3599
3600         return 0;
3601 }
3602
3603 int
3604 init_port_dcb_config(portid_t pid,
3605                      enum dcb_mode_enable dcb_mode,
3606                      enum rte_eth_nb_tcs num_tcs,
3607                      uint8_t pfc_en)
3608 {
3609         struct rte_eth_conf port_conf;
3610         struct rte_port *rte_port;
3611         int retval;
3612         uint16_t i;
3613
3614         rte_port = &ports[pid];
3615
3616         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3617
3618         port_conf.rxmode = rte_port->dev_conf.rxmode;
3619         port_conf.txmode = rte_port->dev_conf.txmode;
3620
3621         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3622         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3623         if (retval < 0)
3624                 return retval;
3625         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3626
3627         /* re-configure the device . */
3628         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3629         if (retval < 0)
3630                 return retval;
3631
3632         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3633         if (retval != 0)
3634                 return retval;
3635
3636         /* If dev_info.vmdq_pool_base is greater than 0,
3637          * the queue id of vmdq pools is started after pf queues.
3638          */
3639         if (dcb_mode == DCB_VT_ENABLED &&
3640             rte_port->dev_info.vmdq_pool_base > 0) {
3641                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3642                         " for port %d.", pid);
3643                 return -1;
3644         }
3645
3646         /* Assume the ports in testpmd have the same dcb capability
3647          * and has the same number of rxq and txq in dcb mode
3648          */
3649         if (dcb_mode == DCB_VT_ENABLED) {
3650                 if (rte_port->dev_info.max_vfs > 0) {
3651                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3652                         nb_txq = rte_port->dev_info.nb_tx_queues;
3653                 } else {
3654                         nb_rxq = rte_port->dev_info.max_rx_queues;
3655                         nb_txq = rte_port->dev_info.max_tx_queues;
3656                 }
3657         } else {
3658                 /*if vt is disabled, use all pf queues */
3659                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3660                         nb_rxq = rte_port->dev_info.max_rx_queues;
3661                         nb_txq = rte_port->dev_info.max_tx_queues;
3662                 } else {
3663                         nb_rxq = (queueid_t)num_tcs;
3664                         nb_txq = (queueid_t)num_tcs;
3665
3666                 }
3667         }
3668         rx_free_thresh = 64;
3669
3670         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3671
3672         rxtx_port_config(rte_port);
3673         /* VLAN filter */
3674         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3675         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3676                 rx_vft_set(pid, vlan_tags[i], 1);
3677
3678         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3679         if (retval != 0)
3680                 return retval;
3681
3682         rte_port->dcb_flag = 1;
3683
3684         /* Enter DCB configuration status */
3685         dcb_config = 1;
3686
3687         return 0;
3688 }
3689
3690 static void
3691 init_port(void)
3692 {
3693         int i;
3694
3695         /* Configuration of Ethernet ports. */
3696         ports = rte_zmalloc("testpmd: ports",
3697                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3698                             RTE_CACHE_LINE_SIZE);
3699         if (ports == NULL) {
3700                 rte_exit(EXIT_FAILURE,
3701                                 "rte_zmalloc(%d struct rte_port) failed\n",
3702                                 RTE_MAX_ETHPORTS);
3703         }
3704         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3705                 LIST_INIT(&ports[i].flow_tunnel_list);
3706         /* Initialize ports NUMA structures */
3707         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3708         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3709         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3710 }
3711
3712 static void
3713 force_quit(void)
3714 {
3715         pmd_test_exit();
3716         prompt_exit();
3717 }
3718
3719 static void
3720 print_stats(void)
3721 {
3722         uint8_t i;
3723         const char clr[] = { 27, '[', '2', 'J', '\0' };
3724         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3725
3726         /* Clear screen and move to top left */
3727         printf("%s%s", clr, top_left);
3728
3729         printf("\nPort statistics ====================================");
3730         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3731                 nic_stats_display(fwd_ports_ids[i]);
3732
3733         fflush(stdout);
3734 }
3735
3736 static void
3737 signal_handler(int signum)
3738 {
3739         if (signum == SIGINT || signum == SIGTERM) {
3740                 printf("\nSignal %d received, preparing to exit...\n",
3741                                 signum);
3742 #ifdef RTE_LIB_PDUMP
3743                 /* uninitialize packet capture framework */
3744                 rte_pdump_uninit();
3745 #endif
3746 #ifdef RTE_LIB_LATENCYSTATS
3747                 if (latencystats_enabled != 0)
3748                         rte_latencystats_uninit();
3749 #endif
3750                 force_quit();
3751                 /* Set flag to indicate the force termination. */
3752                 f_quit = 1;
3753                 /* exit with the expected status */
3754                 signal(signum, SIG_DFL);
3755                 kill(getpid(), signum);
3756         }
3757 }
3758
3759 int
3760 main(int argc, char** argv)
3761 {
3762         int diag;
3763         portid_t port_id;
3764         uint16_t count;
3765         int ret;
3766
3767         signal(SIGINT, signal_handler);
3768         signal(SIGTERM, signal_handler);
3769
3770         testpmd_logtype = rte_log_register("testpmd");
3771         if (testpmd_logtype < 0)
3772                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3773         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3774
3775         diag = rte_eal_init(argc, argv);
3776         if (diag < 0)
3777                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3778                          rte_strerror(rte_errno));
3779
3780         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3781                 rte_exit(EXIT_FAILURE,
3782                          "Secondary process type not supported.\n");
3783
3784         ret = register_eth_event_callback();
3785         if (ret != 0)
3786                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3787
3788 #ifdef RTE_LIB_PDUMP
3789         /* initialize packet capture framework */
3790         rte_pdump_init();
3791 #endif
3792
3793         count = 0;
3794         RTE_ETH_FOREACH_DEV(port_id) {
3795                 ports_ids[count] = port_id;
3796                 count++;
3797         }
3798         nb_ports = (portid_t) count;
3799         if (nb_ports == 0)
3800                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3801
3802         /* allocate port structures, and init them */
3803         init_port();
3804
3805         set_def_fwd_config();
3806         if (nb_lcores == 0)
3807                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3808                          "Check the core mask argument\n");
3809
3810         /* Bitrate/latency stats disabled by default */
3811 #ifdef RTE_LIB_BITRATESTATS
3812         bitrate_enabled = 0;
3813 #endif
3814 #ifdef RTE_LIB_LATENCYSTATS
3815         latencystats_enabled = 0;
3816 #endif
3817
3818         /* on FreeBSD, mlockall() is disabled by default */
3819 #ifdef RTE_EXEC_ENV_FREEBSD
3820         do_mlockall = 0;
3821 #else
3822         do_mlockall = 1;
3823 #endif
3824
3825         argc -= diag;
3826         argv += diag;
3827         if (argc > 1)
3828                 launch_args_parse(argc, argv);
3829
3830         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3831                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3832                         strerror(errno));
3833         }
3834
3835         if (tx_first && interactive)
3836                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3837                                 "interactive mode.\n");
3838
3839         if (tx_first && lsc_interrupt) {
3840                 printf("Warning: lsc_interrupt needs to be off when "
3841                                 " using tx_first. Disabling.\n");
3842                 lsc_interrupt = 0;
3843         }
3844
3845         if (!nb_rxq && !nb_txq)
3846                 printf("Warning: Either rx or tx queues should be non-zero\n");
3847
3848         if (nb_rxq > 1 && nb_rxq > nb_txq)
3849                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3850                        "but nb_txq=%d will prevent to fully test it.\n",
3851                        nb_rxq, nb_txq);
3852
3853         init_config();
3854
3855         if (hot_plug) {
3856                 ret = rte_dev_hotplug_handle_enable();
3857                 if (ret) {
3858                         RTE_LOG(ERR, EAL,
3859                                 "fail to enable hotplug handling.");
3860                         return -1;
3861                 }
3862
3863                 ret = rte_dev_event_monitor_start();
3864                 if (ret) {
3865                         RTE_LOG(ERR, EAL,
3866                                 "fail to start device event monitoring.");
3867                         return -1;
3868                 }
3869
3870                 ret = rte_dev_event_callback_register(NULL,
3871                         dev_event_callback, NULL);
3872                 if (ret) {
3873                         RTE_LOG(ERR, EAL,
3874                                 "fail  to register device event callback\n");
3875                         return -1;
3876                 }
3877         }
3878
3879         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3880                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3881
3882         /* set all ports to promiscuous mode by default */
3883         RTE_ETH_FOREACH_DEV(port_id) {
3884                 ret = rte_eth_promiscuous_enable(port_id);
3885                 if (ret != 0)
3886                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3887                                 port_id, rte_strerror(-ret));
3888         }
3889
3890         /* Init metrics library */
3891         rte_metrics_init(rte_socket_id());
3892
3893 #ifdef RTE_LIB_LATENCYSTATS
3894         if (latencystats_enabled != 0) {
3895                 int ret = rte_latencystats_init(1, NULL);
3896                 if (ret)
3897                         printf("Warning: latencystats init()"
3898                                 " returned error %d\n", ret);
3899                 printf("Latencystats running on lcore %d\n",
3900                         latencystats_lcore_id);
3901         }
3902 #endif
3903
3904         /* Setup bitrate stats */
3905 #ifdef RTE_LIB_BITRATESTATS
3906         if (bitrate_enabled != 0) {
3907                 bitrate_data = rte_stats_bitrate_create();
3908                 if (bitrate_data == NULL)
3909                         rte_exit(EXIT_FAILURE,
3910                                 "Could not allocate bitrate data.\n");
3911                 rte_stats_bitrate_reg(bitrate_data);
3912         }
3913 #endif
3914
3915 #ifdef RTE_LIB_CMDLINE
3916         if (strlen(cmdline_filename) != 0)
3917                 cmdline_read_from_file(cmdline_filename);
3918
3919         if (interactive == 1) {
3920                 if (auto_start) {
3921                         printf("Start automatic packet forwarding\n");
3922                         start_packet_forwarding(0);
3923                 }
3924                 prompt();
3925                 pmd_test_exit();
3926         } else
3927 #endif
3928         {
3929                 char c;
3930                 int rc;
3931
3932                 f_quit = 0;
3933
3934                 printf("No commandline core given, start packet forwarding\n");
3935                 start_packet_forwarding(tx_first);
3936                 if (stats_period != 0) {
3937                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3938                         uint64_t timer_period;
3939
3940                         /* Convert to number of cycles */
3941                         timer_period = stats_period * rte_get_timer_hz();
3942
3943                         while (f_quit == 0) {
3944                                 cur_time = rte_get_timer_cycles();
3945                                 diff_time += cur_time - prev_time;
3946
3947                                 if (diff_time >= timer_period) {
3948                                         print_stats();
3949                                         /* Reset the timer */
3950                                         diff_time = 0;
3951                                 }
3952                                 /* Sleep to avoid unnecessary checks */
3953                                 prev_time = cur_time;
3954                                 sleep(1);
3955                         }
3956                 }
3957
3958                 printf("Press enter to exit\n");
3959                 rc = read(0, &c, 1);
3960                 pmd_test_exit();
3961                 if (rc < 0)
3962                         return 1;
3963         }
3964
3965         ret = rte_eal_cleanup();
3966         if (ret != 0)
3967                 rte_exit(EXIT_FAILURE,
3968                          "EAL cleanup failed: %s\n", strerror(-ret));
3969
3970         return EXIT_SUCCESS;
3971 }