app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_NET_IXGBE
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIB_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIB_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIB_LATENCYSTATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use main core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 219 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 220
 221 /*
 222  * Configuration of packet segments used by the "txonly" processing engine.
 223  */
 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 226         TXONLY_DEF_PACKET_LEN,
 227 };
 228 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 229
 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 231 /**< Split policy for packets to TX. */
 232
 233 uint8_t txonly_multi_flow;
 234 /**< Whether multiple flows are generated in TXONLY mode. */
 235
 236 uint32_t tx_pkt_times_inter;
 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 238
 239 uint32_t tx_pkt_times_intra;
 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 241
 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 245
 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 247 uint8_t dcb_config = 0;
 248
 249 /* Whether the dcb is in testing status */
 250 uint8_t dcb_test = 0;
 251
 252 /*
 253  * Configurable number of RX/TX queues.
 254  */
 255 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 256 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 257 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 258
 259 /*
 260  * Configurable number of RX/TX ring descriptors.
 261  * Defaults are supplied by drivers via ethdev.
 262  */
 263 #define RTE_TEST_RX_DESC_DEFAULT 0
 264 #define RTE_TEST_TX_DESC_DEFAULT 0
 265 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 266 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 267
 268 #define RTE_PMD_PARAM_UNSET -1
 269 /*
 270  * Configurable values of RX and TX ring threshold registers.
 271  */
 272
 273 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 274 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 275 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 276
 277 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 279 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 280
 281 /*
 282  * Configurable value of RX free threshold.
 283  */
 284 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 285
 286 /*
 287  * Configurable value of RX drop enable.
 288  */
 289 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 290
 291 /*
 292  * Configurable value of TX free threshold.
 293  */
 294 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 295
 296 /*
 297  * Configurable value of TX RS bit threshold.
 298  */
 299 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 300
 301 /*
 302  * Configurable value of buffered packets before sending.
 303  */
 304 uint16_t noisy_tx_sw_bufsz;
 305
 306 /*
 307  * Configurable value of packet buffer timeout.
 308  */
 309 uint16_t noisy_tx_sw_buf_flush_time;
 310
 311 /*
 312  * Configurable value for size of VNF internal memory area
 313  * used for simulating noisy neighbour behaviour
 314  */
 315 uint64_t noisy_lkup_mem_sz;
 316
 317 /*
 318  * Configurable value of number of random writes done in
 319  * VNF simulation memory area.
 320  */
 321 uint64_t noisy_lkup_num_writes;
 322
 323 /*
 324  * Configurable value of number of random reads done in
 325  * VNF simulation memory area.
 326  */
 327 uint64_t noisy_lkup_num_reads;
 328
 329 /*
 330  * Configurable value of number of random reads/writes done in
 331  * VNF simulation memory area.
 332  */
 333 uint64_t noisy_lkup_num_reads_writes;
 334
 335 /*
 336  * Receive Side Scaling (RSS) configuration.
 337  */
 338 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 339
 340 /*
 341  * Port topology configuration
 342  */
 343 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 344
 345 /*
 346  * Avoids to flush all the RX streams before starts forwarding.
 347  */
 348 uint8_t no_flush_rx = 0; /* flush by default */
 349
 350 /*
 351  * Flow API isolated mode.
 352  */
 353 uint8_t flow_isolate_all;
 354
 355 /*
 356  * Avoids to check link status when starting/stopping a port.
 357  */
 358 uint8_t no_link_check = 0; /* check by default */
 359
 360 /*
 361  * Don't automatically start all ports in interactive mode.
 362  */
 363 uint8_t no_device_start = 0;
 364
 365 /*
 366  * Enable link status change notification
 367  */
 368 uint8_t lsc_interrupt = 1; /* enabled by default */
 369
 370 /*
 371  * Enable device removal notification.
 372  */
 373 uint8_t rmv_interrupt = 1; /* enabled by default */
 374
 375 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 376
 377 /* After attach, port setup is called on event or by iterator */
 378 bool setup_on_probe_event = true;
 379
 380 /* Clear ptypes on port initialization. */
 381 uint8_t clear_ptypes = true;
 382
 383 /* Hairpin ports configuration mode. */
 384 uint16_t hairpin_mode;
 385
 386 /* Pretty printing of ethdev events */
 387 static const char * const eth_event_desc[] = {
 388         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 389         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 390         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 391         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 392         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 393         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 394         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 395         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 396         [RTE_ETH_EVENT_NEW] = "device probed",
 397         [RTE_ETH_EVENT_DESTROY] = "device released",
 398         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 399         [RTE_ETH_EVENT_MAX] = NULL,
 400 };
 401
 402 /*
 403  * Display or mask ether events
 404  * Default to all events except VF_MBOX
 405  */
 406 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 414 /*
 415  * Decide if all memory are locked for performance.
 416  */
 417 int do_mlockall = 0;
 418
 419 /*
 420  * NIC bypass mode configuration options.
 421  */
 422
 423 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 424 /* The NIC bypass watchdog timeout. */
 425 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 426 #endif
 427
 428
 429 #ifdef RTE_LIB_LATENCYSTATS
 430
 431 /*
 432  * Set when latency stats is enabled in the commandline
 433  */
 434 uint8_t latencystats_enabled;
 435
 436 /*
 437  * Lcore ID to serive latency statistics.
 438  */
 439 lcoreid_t latencystats_lcore_id = -1;
 440
 441 #endif
 442
 443 /*
 444  * Ethernet device configuration.
 445  */
 446 struct rte_eth_rxmode rx_mode = {
 447         /* Default maximum frame length.
 448          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 449          * in init_config().
 450          */
 451         .max_rx_pkt_len = 0,
 452 };
 453
 454 struct rte_eth_txmode tx_mode = {
 455         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 456 };
 457
 458 struct rte_fdir_conf fdir_conf = {
 459         .mode = RTE_FDIR_MODE_NONE,
 460         .pballoc = RTE_FDIR_PBALLOC_64K,
 461         .status = RTE_FDIR_REPORT_STATUS,
 462         .mask = {
 463                 .vlan_tci_mask = 0xFFEF,
 464                 .ipv4_mask     = {
 465                         .src_ip = 0xFFFFFFFF,
 466                         .dst_ip = 0xFFFFFFFF,
 467                 },
 468                 .ipv6_mask     = {
 469                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 470                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 471                 },
 472                 .src_port_mask = 0xFFFF,
 473                 .dst_port_mask = 0xFFFF,
 474                 .mac_addr_byte_mask = 0xFF,
 475                 .tunnel_type_mask = 1,
 476                 .tunnel_id_mask = 0xFFFFFFFF,
 477         },
 478         .drop_queue = 127,
 479 };
 480
 481 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 482
 483 /*
 484  * Display zero values by default for xstats
 485  */
 486 uint8_t xstats_hide_zero;
 487
 488 /*
 489  * Measure of CPU cycles disabled by default
 490  */
 491 uint8_t record_core_cycles;
 492
 493 /*
 494  * Display of RX and TX bursts disabled by default
 495  */
 496 uint8_t record_burst_stats;
 497
 498 unsigned int num_sockets = 0;
 499 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 500
 501 #ifdef RTE_LIB_BITRATESTATS
 502 /* Bitrate statistics */
 503 struct rte_stats_bitrates *bitrate_data;
 504 lcoreid_t bitrate_lcore_id;
 505 uint8_t bitrate_enabled;
 506 #endif
 507
 508 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 509 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 510
 511 /*
 512  * hexadecimal bitmask of RX mq mode can be enabled.
 513  */
 514 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 515
 516 /* Forward function declarations */
 517 static void setup_attached_port(portid_t pi);
 518 static void check_all_ports_link_status(uint32_t port_mask);
 519 static int eth_event_callback(portid_t port_id,
 520                               enum rte_eth_event_type type,
 521                               void *param, void *ret_param);
 522 static void dev_event_callback(const char *device_name,
 523                                 enum rte_dev_event_type type,
 524                                 void *param);
 525
 526 /*
 527  * Check if all the ports are started.
 528  * If yes, return positive value. If not, return zero.
 529  */
 530 static int all_ports_started(void);
 531
 532 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 533 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 534
 535 /* Holds the registered mbuf dynamic flags names. */
 536 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 537
 538 /*
 539  * Helper function to check if socket is already discovered.
 540  * If yes, return positive value. If not, return zero.
 541  */
 542 int
 543 new_socket_id(unsigned int socket_id)
 544 {
 545         unsigned int i;
 546
 547         for (i = 0; i < num_sockets; i++) {
 548                 if (socket_ids[i] == socket_id)
 549                         return 0;
 550         }
 551         return 1;
 552 }
 553
 554 /*
 555  * Setup default configuration.
 556  */
 557 static void
 558 set_default_fwd_lcores_config(void)
 559 {
 560         unsigned int i;
 561         unsigned int nb_lc;
 562         unsigned int sock_num;
 563
 564         nb_lc = 0;
 565         for (i = 0; i < RTE_MAX_LCORE; i++) {
 566                 if (!rte_lcore_is_enabled(i))
 567                         continue;
 568                 sock_num = rte_lcore_to_socket_id(i);
 569                 if (new_socket_id(sock_num)) {
 570                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 571                                 rte_exit(EXIT_FAILURE,
 572                                          "Total sockets greater than %u\n",
 573                                          RTE_MAX_NUMA_NODES);
 574                         }
 575                         socket_ids[num_sockets++] = sock_num;
 576                 }
 577                 if (i == rte_get_main_lcore())
 578                         continue;
 579                 fwd_lcores_cpuids[nb_lc++] = i;
 580         }
 581         nb_lcores = (lcoreid_t) nb_lc;
 582         nb_cfg_lcores = nb_lcores;
 583         nb_fwd_lcores = 1;
 584 }
 585
 586 static void
 587 set_def_peer_eth_addrs(void)
 588 {
 589         portid_t i;
 590
 591         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 592                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 593                 peer_eth_addrs[i].addr_bytes[5] = i;
 594         }
 595 }
 596
 597 static void
 598 set_default_fwd_ports_config(void)
 599 {
 600         portid_t pt_id;
 601         int i = 0;
 602
 603         RTE_ETH_FOREACH_DEV(pt_id) {
 604                 fwd_ports_ids[i++] = pt_id;
 605
 606                 /* Update sockets info according to the attached device */
 607                 int socket_id = rte_eth_dev_socket_id(pt_id);
 608                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 609                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 610                                 rte_exit(EXIT_FAILURE,
 611                                          "Total sockets greater than %u\n",
 612                                          RTE_MAX_NUMA_NODES);
 613                         }
 614                         socket_ids[num_sockets++] = socket_id;
 615                 }
 616         }
 617
 618         nb_cfg_ports = nb_ports;
 619         nb_fwd_ports = nb_ports;
 620 }
 621
 622 void
 623 set_def_fwd_config(void)
 624 {
 625         set_default_fwd_lcores_config();
 626         set_def_peer_eth_addrs();
 627         set_default_fwd_ports_config();
 628 }
 629
 630 /* extremely pessimistic estimation of memory required to create a mempool */
 631 static int
 632 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 633 {
 634         unsigned int n_pages, mbuf_per_pg, leftover;
 635         uint64_t total_mem, mbuf_mem, obj_sz;
 636
 637         /* there is no good way to predict how much space the mempool will
 638          * occupy because it will allocate chunks on the fly, and some of those
 639          * will come from default DPDK memory while some will come from our
 640          * external memory, so just assume 128MB will be enough for everyone.
 641          */
 642         uint64_t hdr_mem = 128 << 20;
 643
 644         /* account for possible non-contiguousness */
 645         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 646         if (obj_sz > pgsz) {
 647                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 648                 return -1;
 649         }
 650
 651         mbuf_per_pg = pgsz / obj_sz;
 652         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 653         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 654
 655         mbuf_mem = n_pages * pgsz;
 656
 657         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 658
 659         if (total_mem > SIZE_MAX) {
 660                 TESTPMD_LOG(ERR, "Memory size too big\n");
 661                 return -1;
 662         }
 663         *out = (size_t)total_mem;
 664
 665         return 0;
 666 }
 667
 668 static int
 669 pagesz_flags(uint64_t page_sz)
 670 {
 671         /* as per mmap() manpage, all page sizes are log2 of page size
 672          * shifted by MAP_HUGE_SHIFT
 673          */
 674         int log2 = rte_log2_u64(page_sz);
 675
 676         return (log2 << HUGE_SHIFT);
 677 }
 678
 679 static void *
 680 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 681 {
 682         void *addr;
 683         int flags;
 684
 685         /* allocate anonymous hugepages */
 686         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 687         if (huge)
 688                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 689
 690         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 691         if (addr == MAP_FAILED)
 692                 return NULL;
 693
 694         return addr;
 695 }
 696
 697 struct extmem_param {
 698         void *addr;
 699         size_t len;
 700         size_t pgsz;
 701         rte_iova_t *iova_table;
 702         unsigned int iova_table_len;
 703 };
 704
 705 static int
 706 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 707                 bool huge)
 708 {
 709         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 710                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 711         unsigned int cur_page, n_pages, pgsz_idx;
 712         size_t mem_sz, cur_pgsz;
 713         rte_iova_t *iovas = NULL;
 714         void *addr;
 715         int ret;
 716
 717         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 718                 /* skip anything that is too big */
 719                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 720                         continue;
 721
 722                 cur_pgsz = pgsizes[pgsz_idx];
 723
 724                 /* if we were told not to allocate hugepages, override */
 725                 if (!huge)
 726                         cur_pgsz = sysconf(_SC_PAGESIZE);
 727
 728                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 729                 if (ret < 0) {
 730                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 731                         return -1;
 732                 }
 733
 734                 /* allocate our memory */
 735                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 736
 737                 /* if we couldn't allocate memory with a specified page size,
 738                  * that doesn't mean we can't do it with other page sizes, so
 739                  * try another one.
 740                  */
 741                 if (addr == NULL)
 742                         continue;
 743
 744                 /* store IOVA addresses for every page in this memory area */
 745                 n_pages = mem_sz / cur_pgsz;
 746
 747                 iovas = malloc(sizeof(*iovas) * n_pages);
 748
 749                 if (iovas == NULL) {
 750                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 751                         goto fail;
 752                 }
 753                 /* lock memory if it's not huge pages */
 754                 if (!huge)
 755                         mlock(addr, mem_sz);
 756
 757                 /* populate IOVA addresses */
 758                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 759                         rte_iova_t iova;
 760                         size_t offset;
 761                         void *cur;
 762
 763                         offset = cur_pgsz * cur_page;
 764                         cur = RTE_PTR_ADD(addr, offset);
 765
 766                         /* touch the page before getting its IOVA */
 767                         *(volatile char *)cur = 0;
 768
 769                         iova = rte_mem_virt2iova(cur);
 770
 771                         iovas[cur_page] = iova;
 772                 }
 773
 774                 break;
 775         }
 776         /* if we couldn't allocate anything */
 777         if (iovas == NULL)
 778                 return -1;
 779
 780         param->addr = addr;
 781         param->len = mem_sz;
 782         param->pgsz = cur_pgsz;
 783         param->iova_table = iovas;
 784         param->iova_table_len = n_pages;
 785
 786         return 0;
 787 fail:
 788         if (iovas)
 789                 free(iovas);
 790         if (addr)
 791                 munmap(addr, mem_sz);
 792
 793         return -1;
 794 }
 795
 796 static int
 797 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 798 {
 799         struct extmem_param param;
 800         int socket_id, ret;
 801
 802         memset(&param, 0, sizeof(param));
 803
 804         /* check if our heap exists */
 805         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 806         if (socket_id < 0) {
 807                 /* create our heap */
 808                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 809                 if (ret < 0) {
 810                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 811                         return -1;
 812                 }
 813         }
 814
 815         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 816         if (ret < 0) {
 817                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 818                 return -1;
 819         }
 820
 821         /* we now have a valid memory area, so add it to heap */
 822         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 823                         param.addr, param.len, param.iova_table,
 824                         param.iova_table_len, param.pgsz);
 825
 826         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 827
 828         /* not needed any more */
 829         free(param.iova_table);
 830
 831         if (ret < 0) {
 832                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 833                 munmap(param.addr, param.len);
 834                 return -1;
 835         }
 836
 837         /* success */
 838
 839         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 840                         param.len >> 20);
 841
 842         return 0;
 843 }
 844 static void
 845 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 846              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 847 {
 848         uint16_t pid = 0;
 849         int ret;
 850
 851         RTE_ETH_FOREACH_DEV(pid) {
 852                 struct rte_eth_dev *dev =
 853                         &rte_eth_devices[pid];
 854
 855                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 856                                         memhdr->len);
 857                 if (ret) {
 858                         TESTPMD_LOG(DEBUG,
 859                                     "unable to DMA unmap addr 0x%p "
 860                                     "for device %s\n",
 861                                     memhdr->addr, dev->data->name);
 862                 }
 863         }
 864         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 865         if (ret) {
 866                 TESTPMD_LOG(DEBUG,
 867                             "unable to un-register addr 0x%p\n", memhdr->addr);
 868         }
 869 }
 870
 871 static void
 872 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 873            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 874 {
 875         uint16_t pid = 0;
 876         size_t page_size = sysconf(_SC_PAGESIZE);
 877         int ret;
 878
 879         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 880                                   page_size);
 881         if (ret) {
 882                 TESTPMD_LOG(DEBUG,
 883                             "unable to register addr 0x%p\n", memhdr->addr);
 884                 return;
 885         }
 886         RTE_ETH_FOREACH_DEV(pid) {
 887                 struct rte_eth_dev *dev =
 888                         &rte_eth_devices[pid];
 889
 890                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 891                                       memhdr->len);
 892                 if (ret) {
 893                         TESTPMD_LOG(DEBUG,
 894                                     "unable to DMA map addr 0x%p "
 895                                     "for device %s\n",
 896                                     memhdr->addr, dev->data->name);
 897                 }
 898         }
 899 }
 900
 901 static unsigned int
 902 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 903             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 904 {
 905         struct rte_pktmbuf_extmem *xmem;
 906         unsigned int ext_num, zone_num, elt_num;
 907         uint16_t elt_size;
 908
 909         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 910         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 911         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 912
 913         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 914         if (xmem == NULL) {
 915                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 916                                  "external buffer descriptors\n");
 917                 *ext_mem = NULL;
 918                 return 0;
 919         }
 920         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 921                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 922                 const struct rte_memzone *mz;
 923                 char mz_name[RTE_MEMZONE_NAMESIZE];
 924                 int ret;
 925
 926                 ret = snprintf(mz_name, sizeof(mz_name),
 927                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 928                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 929                         errno = ENAMETOOLONG;
 930                         ext_num = 0;
 931                         break;
 932                 }
 933                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 934                                                  socket_id,
 935                                                  RTE_MEMZONE_IOVA_CONTIG |
 936                                                  RTE_MEMZONE_1GB |
 937                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 938                                                  EXTBUF_ZONE_SIZE);
 939                 if (mz == NULL) {
 940                         /*
 941                          * The caller exits on external buffer creation
 942                          * error, so there is no need to free memzones.
 943                          */
 944                         errno = ENOMEM;
 945                         ext_num = 0;
 946                         break;
 947                 }
 948                 xseg->buf_ptr = mz->addr;
 949                 xseg->buf_iova = mz->iova;
 950                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 951                 xseg->elt_size = elt_size;
 952         }
 953         if (ext_num == 0 && xmem != NULL) {
 954                 free(xmem);
 955                 xmem = NULL;
 956         }
 957         *ext_mem = xmem;
 958         return ext_num;
 959 }
 960
 961 /*
 962  * Configuration initialisation done once at init time.
 963  */
 964 static struct rte_mempool *
 965 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 966                  unsigned int socket_id, uint16_t size_idx)
 967 {
 968         char pool_name[RTE_MEMPOOL_NAMESIZE];
 969         struct rte_mempool *rte_mp = NULL;
 970         uint32_t mb_size;
 971
 972         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 973         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 974
 975         TESTPMD_LOG(INFO,
 976                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 977                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 978
 979         switch (mp_alloc_type) {
 980         case MP_ALLOC_NATIVE:
 981                 {
 982                         /* wrapper to rte_mempool_create() */
 983                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 984                                         rte_mbuf_best_mempool_ops());
 985                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 986                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 987                         break;
 988                 }
 989         case MP_ALLOC_ANON:
 990                 {
 991                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 992                                 mb_size, (unsigned int) mb_mempool_cache,
 993                                 sizeof(struct rte_pktmbuf_pool_private),
 994                                 socket_id, mempool_flags);
 995                         if (rte_mp == NULL)
 996                                 goto err;
 997
 998                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 999                                 rte_mempool_free(rte_mp);
1000                                 rte_mp = NULL;
1001                                 goto err;
1002                         }
1003                         rte_pktmbuf_pool_init(rte_mp, NULL);
1004                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1005                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1006                         break;
1007                 }
1008         case MP_ALLOC_XMEM:
1009         case MP_ALLOC_XMEM_HUGE:
1010                 {
1011                         int heap_socket;
1012                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1013
1014                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1015                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1016
1017                         heap_socket =
1018                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1019                         if (heap_socket < 0)
1020                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1021
1022                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1023                                         rte_mbuf_best_mempool_ops());
1024                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1025                                         mb_mempool_cache, 0, mbuf_seg_size,
1026                                         heap_socket);
1027                         break;
1028                 }
1029         case MP_ALLOC_XBUF:
1030                 {
1031                         struct rte_pktmbuf_extmem *ext_mem;
1032                         unsigned int ext_num;
1033
1034                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1035                                                socket_id, pool_name, &ext_mem);
1036                         if (ext_num == 0)
1037                                 rte_exit(EXIT_FAILURE,
1038                                          "Can't create pinned data buffers\n");
1039
1040                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1041                                         rte_mbuf_best_mempool_ops());
1042                         rte_mp = rte_pktmbuf_pool_create_extbuf
1043                                         (pool_name, nb_mbuf, mb_mempool_cache,
1044                                          0, mbuf_seg_size, socket_id,
1045                                          ext_mem, ext_num);
1046                         free(ext_mem);
1047                         break;
1048                 }
1049         default:
1050                 {
1051                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1052                 }
1053         }
1054
1055 err:
1056         if (rte_mp == NULL) {
1057                 rte_exit(EXIT_FAILURE,
1058                         "Creation of mbuf pool for socket %u failed: %s\n",
1059                         socket_id, rte_strerror(rte_errno));
1060         } else if (verbose_level > 0) {
1061                 rte_mempool_dump(stdout, rte_mp);
1062         }
1063         return rte_mp;
1064 }
1065
1066 /*
1067  * Check given socket id is valid or not with NUMA mode,
1068  * if valid, return 0, else return -1
1069  */
1070 static int
1071 check_socket_id(const unsigned int socket_id)
1072 {
1073         static int warning_once = 0;
1074
1075         if (new_socket_id(socket_id)) {
1076                 if (!warning_once && numa_support)
1077                         printf("Warning: NUMA should be configured manually by"
1078                                " using --port-numa-config and"
1079                                " --ring-numa-config parameters along with"
1080                                " --numa.\n");
1081                 warning_once = 1;
1082                 return -1;
1083         }
1084         return 0;
1085 }
1086
1087 /*
1088  * Get the allowed maximum number of RX queues.
1089  * *pid return the port id which has minimal value of
1090  * max_rx_queues in all ports.
1091  */
1092 queueid_t
1093 get_allowed_max_nb_rxq(portid_t *pid)
1094 {
1095         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1096         bool max_rxq_valid = false;
1097         portid_t pi;
1098         struct rte_eth_dev_info dev_info;
1099
1100         RTE_ETH_FOREACH_DEV(pi) {
1101                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1102                         continue;
1103
1104                 max_rxq_valid = true;
1105                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1106                         allowed_max_rxq = dev_info.max_rx_queues;
1107                         *pid = pi;
1108                 }
1109         }
1110         return max_rxq_valid ? allowed_max_rxq : 0;
1111 }
1112
1113 /*
1114  * Check input rxq is valid or not.
1115  * If input rxq is not greater than any of maximum number
1116  * of RX queues of all ports, it is valid.
1117  * if valid, return 0, else return -1
1118  */
1119 int
1120 check_nb_rxq(queueid_t rxq)
1121 {
1122         queueid_t allowed_max_rxq;
1123         portid_t pid = 0;
1124
1125         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1126         if (rxq > allowed_max_rxq) {
1127                 printf("Fail: input rxq (%u) can't be greater "
1128                        "than max_rx_queues (%u) of port %u\n",
1129                        rxq,
1130                        allowed_max_rxq,
1131                        pid);
1132                 return -1;
1133         }
1134         return 0;
1135 }
1136
1137 /*
1138  * Get the allowed maximum number of TX queues.
1139  * *pid return the port id which has minimal value of
1140  * max_tx_queues in all ports.
1141  */
1142 queueid_t
1143 get_allowed_max_nb_txq(portid_t *pid)
1144 {
1145         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1146         bool max_txq_valid = false;
1147         portid_t pi;
1148         struct rte_eth_dev_info dev_info;
1149
1150         RTE_ETH_FOREACH_DEV(pi) {
1151                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1152                         continue;
1153
1154                 max_txq_valid = true;
1155                 if (dev_info.max_tx_queues < allowed_max_txq) {
1156                         allowed_max_txq = dev_info.max_tx_queues;
1157                         *pid = pi;
1158                 }
1159         }
1160         return max_txq_valid ? allowed_max_txq : 0;
1161 }
1162
1163 /*
1164  * Check input txq is valid or not.
1165  * If input txq is not greater than any of maximum number
1166  * of TX queues of all ports, it is valid.
1167  * if valid, return 0, else return -1
1168  */
1169 int
1170 check_nb_txq(queueid_t txq)
1171 {
1172         queueid_t allowed_max_txq;
1173         portid_t pid = 0;
1174
1175         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1176         if (txq > allowed_max_txq) {
1177                 printf("Fail: input txq (%u) can't be greater "
1178                        "than max_tx_queues (%u) of port %u\n",
1179                        txq,
1180                        allowed_max_txq,
1181                        pid);
1182                 return -1;
1183         }
1184         return 0;
1185 }
1186
1187 /*
1188  * Get the allowed maximum number of RXDs of every rx queue.
1189  * *pid return the port id which has minimal value of
1190  * max_rxd in all queues of all ports.
1191  */
1192 static uint16_t
1193 get_allowed_max_nb_rxd(portid_t *pid)
1194 {
1195         uint16_t allowed_max_rxd = UINT16_MAX;
1196         portid_t pi;
1197         struct rte_eth_dev_info dev_info;
1198
1199         RTE_ETH_FOREACH_DEV(pi) {
1200                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1201                         continue;
1202
1203                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1204                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1205                         *pid = pi;
1206                 }
1207         }
1208         return allowed_max_rxd;
1209 }
1210
1211 /*
1212  * Get the allowed minimal number of RXDs of every rx queue.
1213  * *pid return the port id which has minimal value of
1214  * min_rxd in all queues of all ports.
1215  */
1216 static uint16_t
1217 get_allowed_min_nb_rxd(portid_t *pid)
1218 {
1219         uint16_t allowed_min_rxd = 0;
1220         portid_t pi;
1221         struct rte_eth_dev_info dev_info;
1222
1223         RTE_ETH_FOREACH_DEV(pi) {
1224                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1225                         continue;
1226
1227                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1228                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1229                         *pid = pi;
1230                 }
1231         }
1232
1233         return allowed_min_rxd;
1234 }
1235
1236 /*
1237  * Check input rxd is valid or not.
1238  * If input rxd is not greater than any of maximum number
1239  * of RXDs of every Rx queues and is not less than any of
1240  * minimal number of RXDs of every Rx queues, it is valid.
1241  * if valid, return 0, else return -1
1242  */
1243 int
1244 check_nb_rxd(queueid_t rxd)
1245 {
1246         uint16_t allowed_max_rxd;
1247         uint16_t allowed_min_rxd;
1248         portid_t pid = 0;
1249
1250         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1251         if (rxd > allowed_max_rxd) {
1252                 printf("Fail: input rxd (%u) can't be greater "
1253                        "than max_rxds (%u) of port %u\n",
1254                        rxd,
1255                        allowed_max_rxd,
1256                        pid);
1257                 return -1;
1258         }
1259
1260         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1261         if (rxd < allowed_min_rxd) {
1262                 printf("Fail: input rxd (%u) can't be less "
1263                        "than min_rxds (%u) of port %u\n",
1264                        rxd,
1265                        allowed_min_rxd,
1266                        pid);
1267                 return -1;
1268         }
1269
1270         return 0;
1271 }
1272
1273 /*
1274  * Get the allowed maximum number of TXDs of every rx queues.
1275  * *pid return the port id which has minimal value of
1276  * max_txd in every tx queue.
1277  */
1278 static uint16_t
1279 get_allowed_max_nb_txd(portid_t *pid)
1280 {
1281         uint16_t allowed_max_txd = UINT16_MAX;
1282         portid_t pi;
1283         struct rte_eth_dev_info dev_info;
1284
1285         RTE_ETH_FOREACH_DEV(pi) {
1286                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1287                         continue;
1288
1289                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1290                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1291                         *pid = pi;
1292                 }
1293         }
1294         return allowed_max_txd;
1295 }
1296
1297 /*
1298  * Get the allowed maximum number of TXDs of every tx queues.
1299  * *pid return the port id which has minimal value of
1300  * min_txd in every tx queue.
1301  */
1302 static uint16_t
1303 get_allowed_min_nb_txd(portid_t *pid)
1304 {
1305         uint16_t allowed_min_txd = 0;
1306         portid_t pi;
1307         struct rte_eth_dev_info dev_info;
1308
1309         RTE_ETH_FOREACH_DEV(pi) {
1310                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1311                         continue;
1312
1313                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1314                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1315                         *pid = pi;
1316                 }
1317         }
1318
1319         return allowed_min_txd;
1320 }
1321
1322 /*
1323  * Check input txd is valid or not.
1324  * If input txd is not greater than any of maximum number
1325  * of TXDs of every Rx queues, it is valid.
1326  * if valid, return 0, else return -1
1327  */
1328 int
1329 check_nb_txd(queueid_t txd)
1330 {
1331         uint16_t allowed_max_txd;
1332         uint16_t allowed_min_txd;
1333         portid_t pid = 0;
1334
1335         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1336         if (txd > allowed_max_txd) {
1337                 printf("Fail: input txd (%u) can't be greater "
1338                        "than max_txds (%u) of port %u\n",
1339                        txd,
1340                        allowed_max_txd,
1341                        pid);
1342                 return -1;
1343         }
1344
1345         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1346         if (txd < allowed_min_txd) {
1347                 printf("Fail: input txd (%u) can't be less "
1348                        "than min_txds (%u) of port %u\n",
1349                        txd,
1350                        allowed_min_txd,
1351                        pid);
1352                 return -1;
1353         }
1354         return 0;
1355 }
1356
1357
1358 /*
1359  * Get the allowed maximum number of hairpin queues.
1360  * *pid return the port id which has minimal value of
1361  * max_hairpin_queues in all ports.
1362  */
1363 queueid_t
1364 get_allowed_max_nb_hairpinq(portid_t *pid)
1365 {
1366         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1367         portid_t pi;
1368         struct rte_eth_hairpin_cap cap;
1369
1370         RTE_ETH_FOREACH_DEV(pi) {
1371                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1372                         *pid = pi;
1373                         return 0;
1374                 }
1375                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1376                         allowed_max_hairpinq = cap.max_nb_queues;
1377                         *pid = pi;
1378                 }
1379         }
1380         return allowed_max_hairpinq;
1381 }
1382
1383 /*
1384  * Check input hairpin is valid or not.
1385  * If input hairpin is not greater than any of maximum number
1386  * of hairpin queues of all ports, it is valid.
1387  * if valid, return 0, else return -1
1388  */
1389 int
1390 check_nb_hairpinq(queueid_t hairpinq)
1391 {
1392         queueid_t allowed_max_hairpinq;
1393         portid_t pid = 0;
1394
1395         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1396         if (hairpinq > allowed_max_hairpinq) {
1397                 printf("Fail: input hairpin (%u) can't be greater "
1398                        "than max_hairpin_queues (%u) of port %u\n",
1399                        hairpinq, allowed_max_hairpinq, pid);
1400                 return -1;
1401         }
1402         return 0;
1403 }
1404
1405 static void
1406 init_config(void)
1407 {
1408         portid_t pid;
1409         struct rte_port *port;
1410         struct rte_mempool *mbp;
1411         unsigned int nb_mbuf_per_pool;
1412         lcoreid_t  lc_id;
1413         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1414         struct rte_gro_param gro_param;
1415         uint32_t gso_types;
1416         uint16_t data_size;
1417         bool warning = 0;
1418         int k;
1419         int ret;
1420
1421         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1422
1423         /* Configuration of logical cores. */
1424         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1425                                 sizeof(struct fwd_lcore *) * nb_lcores,
1426                                 RTE_CACHE_LINE_SIZE);
1427         if (fwd_lcores == NULL) {
1428                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1429                                                         "failed\n", nb_lcores);
1430         }
1431         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1432                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1433                                                sizeof(struct fwd_lcore),
1434                                                RTE_CACHE_LINE_SIZE);
1435                 if (fwd_lcores[lc_id] == NULL) {
1436                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1437                                                                 "failed\n");
1438                 }
1439                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1440         }
1441
1442         RTE_ETH_FOREACH_DEV(pid) {
1443                 port = &ports[pid];
1444                 /* Apply default TxRx configuration for all ports */
1445                 port->dev_conf.txmode = tx_mode;
1446                 port->dev_conf.rxmode = rx_mode;
1447
1448                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1449                 if (ret != 0)
1450                         rte_exit(EXIT_FAILURE,
1451                                  "rte_eth_dev_info_get() failed\n");
1452
1453                 ret = update_jumbo_frame_offload(pid);
1454                 if (ret != 0)
1455                         printf("Updating jumbo frame offload failed for port %u\n",
1456                                 pid);
1457
1458                 if (!(port->dev_info.tx_offload_capa &
1459                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1460                         port->dev_conf.txmode.offloads &=
1461                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1462                 if (numa_support) {
1463                         if (port_numa[pid] != NUMA_NO_CONFIG)
1464                                 port_per_socket[port_numa[pid]]++;
1465                         else {
1466                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1467
1468                                 /*
1469                                  * if socket_id is invalid,
1470                                  * set to the first available socket.
1471                                  */
1472                                 if (check_socket_id(socket_id) < 0)
1473                                         socket_id = socket_ids[0];
1474                                 port_per_socket[socket_id]++;
1475                         }
1476                 }
1477
1478                 /* Apply Rx offloads configuration */
1479                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1480                         port->rx_conf[k].offloads =
1481                                 port->dev_conf.rxmode.offloads;
1482                 /* Apply Tx offloads configuration */
1483                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1484                         port->tx_conf[k].offloads =
1485                                 port->dev_conf.txmode.offloads;
1486
1487                 /* set flag to initialize port/queue */
1488                 port->need_reconfig = 1;
1489                 port->need_reconfig_queues = 1;
1490                 port->tx_metadata = 0;
1491
1492                 /* Check for maximum number of segments per MTU. Accordingly
1493                  * update the mbuf data size.
1494                  */
1495                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1496                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1497                         data_size = rx_mode.max_rx_pkt_len /
1498                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1499
1500                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1501                                                         mbuf_data_size[0]) {
1502                                 mbuf_data_size[0] = data_size +
1503                                                  RTE_PKTMBUF_HEADROOM;
1504                                 warning = 1;
1505                         }
1506                 }
1507         }
1508
1509         if (warning)
1510                 TESTPMD_LOG(WARNING,
1511                             "Configured mbuf size of the first segment %hu\n",
1512                             mbuf_data_size[0]);
1513         /*
1514          * Create pools of mbuf.
1515          * If NUMA support is disabled, create a single pool of mbuf in
1516          * socket 0 memory by default.
1517          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1518          *
1519          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1520          * nb_txd can be configured at run time.
1521          */
1522         if (param_total_num_mbufs)
1523                 nb_mbuf_per_pool = param_total_num_mbufs;
1524         else {
1525                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1526                         (nb_lcores * mb_mempool_cache) +
1527                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1528                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1529         }
1530
1531         if (numa_support) {
1532                 uint8_t i, j;
1533
1534                 for (i = 0; i < num_sockets; i++)
1535                         for (j = 0; j < mbuf_data_size_n; j++)
1536                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1537                                         mbuf_pool_create(mbuf_data_size[j],
1538                                                           nb_mbuf_per_pool,
1539                                                           socket_ids[i], j);
1540         } else {
1541                 uint8_t i;
1542
1543                 for (i = 0; i < mbuf_data_size_n; i++)
1544                         mempools[i] = mbuf_pool_create
1545                                         (mbuf_data_size[i],
1546                                          nb_mbuf_per_pool,
1547                                          socket_num == UMA_NO_CONFIG ?
1548                                          0 : socket_num, i);
1549         }
1550
1551         init_port_config();
1552
1553         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1554                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1555         /*
1556          * Records which Mbuf pool to use by each logical core, if needed.
1557          */
1558         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1559                 mbp = mbuf_pool_find(
1560                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1561
1562                 if (mbp == NULL)
1563                         mbp = mbuf_pool_find(0, 0);
1564                 fwd_lcores[lc_id]->mbp = mbp;
1565                 /* initialize GSO context */
1566                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1567                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1568                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1569                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1570                         RTE_ETHER_CRC_LEN;
1571                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1572         }
1573
1574         /* Configuration of packet forwarding streams. */
1575         if (init_fwd_streams() < 0)
1576                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1577
1578         fwd_config_setup();
1579
1580         /* create a gro context for each lcore */
1581         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1582         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1583         gro_param.max_item_per_flow = MAX_PKT_BURST;
1584         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1585                 gro_param.socket_id = rte_lcore_to_socket_id(
1586                                 fwd_lcores_cpuids[lc_id]);
1587                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1588                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1589                         rte_exit(EXIT_FAILURE,
1590                                         "rte_gro_ctx_create() failed\n");
1591                 }
1592         }
1593 }
1594
1595
1596 void
1597 reconfig(portid_t new_port_id, unsigned socket_id)
1598 {
1599         struct rte_port *port;
1600         int ret;
1601
1602         /* Reconfiguration of Ethernet ports. */
1603         port = &ports[new_port_id];
1604
1605         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1606         if (ret != 0)
1607                 return;
1608
1609         /* set flag to initialize port/queue */
1610         port->need_reconfig = 1;
1611         port->need_reconfig_queues = 1;
1612         port->socket_id = socket_id;
1613
1614         init_port_config();
1615 }
1616
1617
1618 int
1619 init_fwd_streams(void)
1620 {
1621         portid_t pid;
1622         struct rte_port *port;
1623         streamid_t sm_id, nb_fwd_streams_new;
1624         queueid_t q;
1625
1626         /* set socket id according to numa or not */
1627         RTE_ETH_FOREACH_DEV(pid) {
1628                 port = &ports[pid];
1629                 if (nb_rxq > port->dev_info.max_rx_queues) {
1630                         printf("Fail: nb_rxq(%d) is greater than "
1631                                 "max_rx_queues(%d)\n", nb_rxq,
1632                                 port->dev_info.max_rx_queues);
1633                         return -1;
1634                 }
1635                 if (nb_txq > port->dev_info.max_tx_queues) {
1636                         printf("Fail: nb_txq(%d) is greater than "
1637                                 "max_tx_queues(%d)\n", nb_txq,
1638                                 port->dev_info.max_tx_queues);
1639                         return -1;
1640                 }
1641                 if (numa_support) {
1642                         if (port_numa[pid] != NUMA_NO_CONFIG)
1643                                 port->socket_id = port_numa[pid];
1644                         else {
1645                                 port->socket_id = rte_eth_dev_socket_id(pid);
1646
1647                                 /*
1648                                  * if socket_id is invalid,
1649                                  * set to the first available socket.
1650                                  */
1651                                 if (check_socket_id(port->socket_id) < 0)
1652                                         port->socket_id = socket_ids[0];
1653                         }
1654                 }
1655                 else {
1656                         if (socket_num == UMA_NO_CONFIG)
1657                                 port->socket_id = 0;
1658                         else
1659                                 port->socket_id = socket_num;
1660                 }
1661         }
1662
1663         q = RTE_MAX(nb_rxq, nb_txq);
1664         if (q == 0) {
1665                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1666                 return -1;
1667         }
1668         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1669         if (nb_fwd_streams_new == nb_fwd_streams)
1670                 return 0;
1671         /* clear the old */
1672         if (fwd_streams != NULL) {
1673                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1674                         if (fwd_streams[sm_id] == NULL)
1675                                 continue;
1676                         rte_free(fwd_streams[sm_id]);
1677                         fwd_streams[sm_id] = NULL;
1678                 }
1679                 rte_free(fwd_streams);
1680                 fwd_streams = NULL;
1681         }
1682
1683         /* init new */
1684         nb_fwd_streams = nb_fwd_streams_new;
1685         if (nb_fwd_streams) {
1686                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1687                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1688                         RTE_CACHE_LINE_SIZE);
1689                 if (fwd_streams == NULL)
1690                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1691                                  " (struct fwd_stream *)) failed\n",
1692                                  nb_fwd_streams);
1693
1694                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1695                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1696                                 " struct fwd_stream", sizeof(struct fwd_stream),
1697                                 RTE_CACHE_LINE_SIZE);
1698                         if (fwd_streams[sm_id] == NULL)
1699                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1700                                          "(struct fwd_stream) failed\n");
1701                 }
1702         }
1703
1704         return 0;
1705 }
1706
1707 static void
1708 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1709 {
1710         uint64_t total_burst, sburst;
1711         uint64_t nb_burst;
1712         uint64_t burst_stats[4];
1713         uint16_t pktnb_stats[4];
1714         uint16_t nb_pkt;
1715         int burst_percent[4], sburstp;
1716         int i;
1717
1718         /*
1719          * First compute the total number of packet bursts and the
1720          * two highest numbers of bursts of the same number of packets.
1721          */
1722         memset(&burst_stats, 0x0, sizeof(burst_stats));
1723         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1724
1725         /* Show stats for 0 burst size always */
1726         total_burst = pbs->pkt_burst_spread[0];
1727         burst_stats[0] = pbs->pkt_burst_spread[0];
1728         pktnb_stats[0] = 0;
1729
1730         /* Find the next 2 burst sizes with highest occurrences. */
1731         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1732                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1733
1734                 if (nb_burst == 0)
1735                         continue;
1736
1737                 total_burst += nb_burst;
1738
1739                 if (nb_burst > burst_stats[1]) {
1740                         burst_stats[2] = burst_stats[1];
1741                         pktnb_stats[2] = pktnb_stats[1];
1742                         burst_stats[1] = nb_burst;
1743                         pktnb_stats[1] = nb_pkt;
1744                 } else if (nb_burst > burst_stats[2]) {
1745                         burst_stats[2] = nb_burst;
1746                         pktnb_stats[2] = nb_pkt;
1747                 }
1748         }
1749         if (total_burst == 0)
1750                 return;
1751
1752         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1753         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1754                 if (i == 3) {
1755                         printf("%d%% of other]\n", 100 - sburstp);
1756                         return;
1757                 }
1758
1759                 sburst += burst_stats[i];
1760                 if (sburst == total_burst) {
1761                         printf("%d%% of %d pkts]\n",
1762                                 100 - sburstp, (int) pktnb_stats[i]);
1763                         return;
1764                 }
1765
1766                 burst_percent[i] =
1767                         (double)burst_stats[i] / total_burst * 100;
1768                 printf("%d%% of %d pkts + ",
1769                         burst_percent[i], (int) pktnb_stats[i]);
1770                 sburstp += burst_percent[i];
1771         }
1772 }
1773
1774 static void
1775 fwd_stream_stats_display(streamid_t stream_id)
1776 {
1777         struct fwd_stream *fs;
1778         static const char *fwd_top_stats_border = "-------";
1779
1780         fs = fwd_streams[stream_id];
1781         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1782             (fs->fwd_dropped == 0))
1783                 return;
1784         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1785                "TX Port=%2d/Queue=%2d %s\n",
1786                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1787                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1788         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1789                " TX-dropped: %-14"PRIu64,
1790                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1791
1792         /* if checksum mode */
1793         if (cur_fwd_eng == &csum_fwd_engine) {
1794                 printf("  RX- bad IP checksum: %-14"PRIu64
1795                        "  Rx- bad L4 checksum: %-14"PRIu64
1796                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1797                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1798                         fs->rx_bad_outer_l4_csum);
1799                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1800                         fs->rx_bad_outer_ip_csum);
1801         } else {
1802                 printf("\n");
1803         }
1804
1805         if (record_burst_stats) {
1806                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1807                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1808         }
1809 }
1810
1811 void
1812 fwd_stats_display(void)
1813 {
1814         static const char *fwd_stats_border = "----------------------";
1815         static const char *acc_stats_border = "+++++++++++++++";
1816         struct {
1817                 struct fwd_stream *rx_stream;
1818                 struct fwd_stream *tx_stream;
1819                 uint64_t tx_dropped;
1820                 uint64_t rx_bad_ip_csum;
1821                 uint64_t rx_bad_l4_csum;
1822                 uint64_t rx_bad_outer_l4_csum;
1823                 uint64_t rx_bad_outer_ip_csum;
1824         } ports_stats[RTE_MAX_ETHPORTS];
1825         uint64_t total_rx_dropped = 0;
1826         uint64_t total_tx_dropped = 0;
1827         uint64_t total_rx_nombuf = 0;
1828         struct rte_eth_stats stats;
1829         uint64_t fwd_cycles = 0;
1830         uint64_t total_recv = 0;
1831         uint64_t total_xmit = 0;
1832         struct rte_port *port;
1833         streamid_t sm_id;
1834         portid_t pt_id;
1835         int i;
1836
1837         memset(ports_stats, 0, sizeof(ports_stats));
1838
1839         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1840                 struct fwd_stream *fs = fwd_streams[sm_id];
1841
1842                 if (cur_fwd_config.nb_fwd_streams >
1843                     cur_fwd_config.nb_fwd_ports) {
1844                         fwd_stream_stats_display(sm_id);
1845                 } else {
1846                         ports_stats[fs->tx_port].tx_stream = fs;
1847                         ports_stats[fs->rx_port].rx_stream = fs;
1848                 }
1849
1850                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1851
1852                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1853                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1854                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1855                                 fs->rx_bad_outer_l4_csum;
1856                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1857                                 fs->rx_bad_outer_ip_csum;
1858
1859                 if (record_core_cycles)
1860                         fwd_cycles += fs->core_cycles;
1861         }
1862         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1863                 pt_id = fwd_ports_ids[i];
1864                 port = &ports[pt_id];
1865
1866                 rte_eth_stats_get(pt_id, &stats);
1867                 stats.ipackets -= port->stats.ipackets;
1868                 stats.opackets -= port->stats.opackets;
1869                 stats.ibytes -= port->stats.ibytes;
1870                 stats.obytes -= port->stats.obytes;
1871                 stats.imissed -= port->stats.imissed;
1872                 stats.oerrors -= port->stats.oerrors;
1873                 stats.rx_nombuf -= port->stats.rx_nombuf;
1874
1875                 total_recv += stats.ipackets;
1876                 total_xmit += stats.opackets;
1877                 total_rx_dropped += stats.imissed;
1878                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1879                 total_tx_dropped += stats.oerrors;
1880                 total_rx_nombuf  += stats.rx_nombuf;
1881
1882                 printf("\n  %s Forward statistics for port %-2d %s\n",
1883                        fwd_stats_border, pt_id, fwd_stats_border);
1884
1885                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1886                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1887                        stats.ipackets + stats.imissed);
1888
1889                 if (cur_fwd_eng == &csum_fwd_engine) {
1890                         printf("  Bad-ipcsum: %-14"PRIu64
1891                                " Bad-l4csum: %-14"PRIu64
1892                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1893                                ports_stats[pt_id].rx_bad_ip_csum,
1894                                ports_stats[pt_id].rx_bad_l4_csum,
1895                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1896                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1897                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1898                 }
1899                 if (stats.ierrors + stats.rx_nombuf > 0) {
1900                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1901                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1902                 }
1903
1904                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1905                        "TX-total: %-"PRIu64"\n",
1906                        stats.opackets, ports_stats[pt_id].tx_dropped,
1907                        stats.opackets + ports_stats[pt_id].tx_dropped);
1908
1909                 if (record_burst_stats) {
1910                         if (ports_stats[pt_id].rx_stream)
1911                                 pkt_burst_stats_display("RX",
1912                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1913                         if (ports_stats[pt_id].tx_stream)
1914                                 pkt_burst_stats_display("TX",
1915                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1916                 }
1917
1918                 printf("  %s--------------------------------%s\n",
1919                        fwd_stats_border, fwd_stats_border);
1920         }
1921
1922         printf("\n  %s Accumulated forward statistics for all ports"
1923                "%s\n",
1924                acc_stats_border, acc_stats_border);
1925         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1926                "%-"PRIu64"\n"
1927                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1928                "%-"PRIu64"\n",
1929                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1930                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1931         if (total_rx_nombuf > 0)
1932                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1933         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1934                "%s\n",
1935                acc_stats_border, acc_stats_border);
1936         if (record_core_cycles) {
1937 #define CYC_PER_MHZ 1E6
1938                 if (total_recv > 0 || total_xmit > 0) {
1939                         uint64_t total_pkts = 0;
1940                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1941                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1942                                 total_pkts = total_xmit;
1943                         else
1944                                 total_pkts = total_recv;
1945
1946                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1947                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1948                                " MHz Clock\n",
1949                                (double) fwd_cycles / total_pkts,
1950                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1951                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1952                 }
1953         }
1954 }
1955
1956 void
1957 fwd_stats_reset(void)
1958 {
1959         streamid_t sm_id;
1960         portid_t pt_id;
1961         int i;
1962
1963         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1964                 pt_id = fwd_ports_ids[i];
1965                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1966         }
1967         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1968                 struct fwd_stream *fs = fwd_streams[sm_id];
1969
1970                 fs->rx_packets = 0;
1971                 fs->tx_packets = 0;
1972                 fs->fwd_dropped = 0;
1973                 fs->rx_bad_ip_csum = 0;
1974                 fs->rx_bad_l4_csum = 0;
1975                 fs->rx_bad_outer_l4_csum = 0;
1976                 fs->rx_bad_outer_ip_csum = 0;
1977
1978                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1979                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1980                 fs->core_cycles = 0;
1981         }
1982 }
1983
1984 static void
1985 flush_fwd_rx_queues(void)
1986 {
1987         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1988         portid_t  rxp;
1989         portid_t port_id;
1990         queueid_t rxq;
1991         uint16_t  nb_rx;
1992         uint16_t  i;
1993         uint8_t   j;
1994         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1995         uint64_t timer_period;
1996
1997         /* convert to number of cycles */
1998         timer_period = rte_get_timer_hz(); /* 1 second timeout */
1999
2000         for (j = 0; j < 2; j++) {
2001                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2002                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2003                                 port_id = fwd_ports_ids[rxp];
2004                                 /**
2005                                 * testpmd can stuck in the below do while loop
2006                                 * if rte_eth_rx_burst() always returns nonzero
2007                                 * packets. So timer is added to exit this loop
2008                                 * after 1sec timer expiry.
2009                                 */
2010                                 prev_tsc = rte_rdtsc();
2011                                 do {
2012                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2013                                                 pkts_burst, MAX_PKT_BURST);
2014                                         for (i = 0; i < nb_rx; i++)
2015                                                 rte_pktmbuf_free(pkts_burst[i]);
2016
2017                                         cur_tsc = rte_rdtsc();
2018                                         diff_tsc = cur_tsc - prev_tsc;
2019                                         timer_tsc += diff_tsc;
2020                                 } while ((nb_rx > 0) &&
2021                                         (timer_tsc < timer_period));
2022                                 timer_tsc = 0;
2023                         }
2024                 }
2025                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2026         }
2027 }
2028
2029 static void
2030 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2031 {
2032         struct fwd_stream **fsm;
2033         streamid_t nb_fs;
2034         streamid_t sm_id;
2035 #ifdef RTE_LIB_BITRATESTATS
2036         uint64_t tics_per_1sec;
2037         uint64_t tics_datum;
2038         uint64_t tics_current;
2039         uint16_t i, cnt_ports;
2040
2041         cnt_ports = nb_ports;
2042         tics_datum = rte_rdtsc();
2043         tics_per_1sec = rte_get_timer_hz();
2044 #endif
2045         fsm = &fwd_streams[fc->stream_idx];
2046         nb_fs = fc->stream_nb;
2047         do {
2048                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2049                         (*pkt_fwd)(fsm[sm_id]);
2050 #ifdef RTE_LIB_BITRATESTATS
2051                 if (bitrate_enabled != 0 &&
2052                                 bitrate_lcore_id == rte_lcore_id()) {
2053                         tics_current = rte_rdtsc();
2054                         if (tics_current - tics_datum >= tics_per_1sec) {
2055                                 /* Periodic bitrate calculation */
2056                                 for (i = 0; i < cnt_ports; i++)
2057                                         rte_stats_bitrate_calc(bitrate_data,
2058                                                 ports_ids[i]);
2059                                 tics_datum = tics_current;
2060                         }
2061                 }
2062 #endif
2063 #ifdef RTE_LIB_LATENCYSTATS
2064                 if (latencystats_enabled != 0 &&
2065                                 latencystats_lcore_id == rte_lcore_id())
2066                         rte_latencystats_update();
2067 #endif
2068
2069         } while (! fc->stopped);
2070 }
2071
2072 static int
2073 start_pkt_forward_on_core(void *fwd_arg)
2074 {
2075         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2076                              cur_fwd_config.fwd_eng->packet_fwd);
2077         return 0;
2078 }
2079
2080 /*
2081  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2082  * Used to start communication flows in network loopback test configurations.
2083  */
2084 static int
2085 run_one_txonly_burst_on_core(void *fwd_arg)
2086 {
2087         struct fwd_lcore *fwd_lc;
2088         struct fwd_lcore tmp_lcore;
2089
2090         fwd_lc = (struct fwd_lcore *) fwd_arg;
2091         tmp_lcore = *fwd_lc;
2092         tmp_lcore.stopped = 1;
2093         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2094         return 0;
2095 }
2096
2097 /*
2098  * Launch packet forwarding:
2099  *     - Setup per-port forwarding context.
2100  *     - launch logical cores with their forwarding configuration.
2101  */
2102 static void
2103 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2104 {
2105         port_fwd_begin_t port_fwd_begin;
2106         unsigned int i;
2107         unsigned int lc_id;
2108         int diag;
2109
2110         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2111         if (port_fwd_begin != NULL) {
2112                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2113                         (*port_fwd_begin)(fwd_ports_ids[i]);
2114         }
2115         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2116                 lc_id = fwd_lcores_cpuids[i];
2117                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2118                         fwd_lcores[i]->stopped = 0;
2119                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2120                                                      fwd_lcores[i], lc_id);
2121                         if (diag != 0)
2122                                 printf("launch lcore %u failed - diag=%d\n",
2123                                        lc_id, diag);
2124                 }
2125         }
2126 }
2127
2128 /*
2129  * Launch packet forwarding configuration.
2130  */
2131 void
2132 start_packet_forwarding(int with_tx_first)
2133 {
2134         port_fwd_begin_t port_fwd_begin;
2135         port_fwd_end_t  port_fwd_end;
2136         struct rte_port *port;
2137         unsigned int i;
2138         portid_t   pt_id;
2139
2140         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2141                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2142
2143         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2144                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2145
2146         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2147                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2148                 (!nb_rxq || !nb_txq))
2149                 rte_exit(EXIT_FAILURE,
2150                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2151                         cur_fwd_eng->fwd_mode_name);
2152
2153         if (all_ports_started() == 0) {
2154                 printf("Not all ports were started\n");
2155                 return;
2156         }
2157         if (test_done == 0) {
2158                 printf("Packet forwarding already started\n");
2159                 return;
2160         }
2161
2162
2163         if(dcb_test) {
2164                 for (i = 0; i < nb_fwd_ports; i++) {
2165                         pt_id = fwd_ports_ids[i];
2166                         port = &ports[pt_id];
2167                         if (!port->dcb_flag) {
2168                                 printf("In DCB mode, all forwarding ports must "
2169                                        "be configured in this mode.\n");
2170                                 return;
2171                         }
2172                 }
2173                 if (nb_fwd_lcores == 1) {
2174                         printf("In DCB mode,the nb forwarding cores "
2175                                "should be larger than 1.\n");
2176                         return;
2177                 }
2178         }
2179         test_done = 0;
2180
2181         fwd_config_setup();
2182
2183         if(!no_flush_rx)
2184                 flush_fwd_rx_queues();
2185
2186         pkt_fwd_config_display(&cur_fwd_config);
2187         rxtx_config_display();
2188
2189         fwd_stats_reset();
2190         if (with_tx_first) {
2191                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2192                 if (port_fwd_begin != NULL) {
2193                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2194                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2195                 }
2196                 while (with_tx_first--) {
2197                         launch_packet_forwarding(
2198                                         run_one_txonly_burst_on_core);
2199                         rte_eal_mp_wait_lcore();
2200                 }
2201                 port_fwd_end = tx_only_engine.port_fwd_end;
2202                 if (port_fwd_end != NULL) {
2203                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2204                                 (*port_fwd_end)(fwd_ports_ids[i]);
2205                 }
2206         }
2207         launch_packet_forwarding(start_pkt_forward_on_core);
2208 }
2209
2210 void
2211 stop_packet_forwarding(void)
2212 {
2213         port_fwd_end_t port_fwd_end;
2214         lcoreid_t lc_id;
2215         portid_t pt_id;
2216         int i;
2217
2218         if (test_done) {
2219                 printf("Packet forwarding not started\n");
2220                 return;
2221         }
2222         printf("Telling cores to stop...");
2223         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2224                 fwd_lcores[lc_id]->stopped = 1;
2225         printf("\nWaiting for lcores to finish...\n");
2226         rte_eal_mp_wait_lcore();
2227         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2228         if (port_fwd_end != NULL) {
2229                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2230                         pt_id = fwd_ports_ids[i];
2231                         (*port_fwd_end)(pt_id);
2232                 }
2233         }
2234
2235         fwd_stats_display();
2236
2237         printf("\nDone.\n");
2238         test_done = 1;
2239 }
2240
2241 void
2242 dev_set_link_up(portid_t pid)
2243 {
2244         if (rte_eth_dev_set_link_up(pid) < 0)
2245                 printf("\nSet link up fail.\n");
2246 }
2247
2248 void
2249 dev_set_link_down(portid_t pid)
2250 {
2251         if (rte_eth_dev_set_link_down(pid) < 0)
2252                 printf("\nSet link down fail.\n");
2253 }
2254
2255 static int
2256 all_ports_started(void)
2257 {
2258         portid_t pi;
2259         struct rte_port *port;
2260
2261         RTE_ETH_FOREACH_DEV(pi) {
2262                 port = &ports[pi];
2263                 /* Check if there is a port which is not started */
2264                 if ((port->port_status != RTE_PORT_STARTED) &&
2265                         (port->slave_flag == 0))
2266                         return 0;
2267         }
2268
2269         /* No port is not started */
2270         return 1;
2271 }
2272
2273 int
2274 port_is_stopped(portid_t port_id)
2275 {
2276         struct rte_port *port = &ports[port_id];
2277
2278         if ((port->port_status != RTE_PORT_STOPPED) &&
2279             (port->slave_flag == 0))
2280                 return 0;
2281         return 1;
2282 }
2283
2284 int
2285 all_ports_stopped(void)
2286 {
2287         portid_t pi;
2288
2289         RTE_ETH_FOREACH_DEV(pi) {
2290                 if (!port_is_stopped(pi))
2291                         return 0;
2292         }
2293
2294         return 1;
2295 }
2296
2297 int
2298 port_is_started(portid_t port_id)
2299 {
2300         if (port_id_is_invalid(port_id, ENABLED_WARN))
2301                 return 0;
2302
2303         if (ports[port_id].port_status != RTE_PORT_STARTED)
2304                 return 0;
2305
2306         return 1;
2307 }
2308
2309 /* Configure the Rx and Tx hairpin queues for the selected port. */
2310 static int
2311 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2312 {
2313         queueid_t qi;
2314         struct rte_eth_hairpin_conf hairpin_conf = {
2315                 .peer_count = 1,
2316         };
2317         int i;
2318         int diag;
2319         struct rte_port *port = &ports[pi];
2320         uint16_t peer_rx_port = pi;
2321         uint16_t peer_tx_port = pi;
2322         uint32_t manual = 1;
2323         uint32_t tx_exp = hairpin_mode & 0x10;
2324
2325         if (!(hairpin_mode & 0xf)) {
2326                 peer_rx_port = pi;
2327                 peer_tx_port = pi;
2328                 manual = 0;
2329         } else if (hairpin_mode & 0x1) {
2330                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2331                                                        RTE_ETH_DEV_NO_OWNER);
2332                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2333                         peer_tx_port = rte_eth_find_next_owned_by(0,
2334                                                 RTE_ETH_DEV_NO_OWNER);
2335                 if (p_pi != RTE_MAX_ETHPORTS) {
2336                         peer_rx_port = p_pi;
2337                 } else {
2338                         uint16_t next_pi;
2339
2340                         /* Last port will be the peer RX port of the first. */
2341                         RTE_ETH_FOREACH_DEV(next_pi)
2342                                 peer_rx_port = next_pi;
2343                 }
2344                 manual = 1;
2345         } else if (hairpin_mode & 0x2) {
2346                 if (cnt_pi & 0x1) {
2347                         peer_rx_port = p_pi;
2348                 } else {
2349                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2350                                                 RTE_ETH_DEV_NO_OWNER);
2351                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2352                                 peer_rx_port = pi;
2353                 }
2354                 peer_tx_port = peer_rx_port;
2355                 manual = 1;
2356         }
2357
2358         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2359                 hairpin_conf.peers[0].port = peer_rx_port;
2360                 hairpin_conf.peers[0].queue = i + nb_rxq;
2361                 hairpin_conf.manual_bind = !!manual;
2362                 hairpin_conf.tx_explicit = !!tx_exp;
2363                 diag = rte_eth_tx_hairpin_queue_setup
2364                         (pi, qi, nb_txd, &hairpin_conf);
2365                 i++;
2366                 if (diag == 0)
2367                         continue;
2368
2369                 /* Fail to setup rx queue, return */
2370                 if (rte_atomic16_cmpset(&(port->port_status),
2371                                         RTE_PORT_HANDLING,
2372                                         RTE_PORT_STOPPED) == 0)
2373                         printf("Port %d can not be set back "
2374                                         "to stopped\n", pi);
2375                 printf("Fail to configure port %d hairpin "
2376                                 "queues\n", pi);
2377                 /* try to reconfigure queues next time */
2378                 port->need_reconfig_queues = 1;
2379                 return -1;
2380         }
2381         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2382                 hairpin_conf.peers[0].port = peer_tx_port;
2383                 hairpin_conf.peers[0].queue = i + nb_txq;
2384                 hairpin_conf.manual_bind = !!manual;
2385                 hairpin_conf.tx_explicit = !!tx_exp;
2386                 diag = rte_eth_rx_hairpin_queue_setup
2387                         (pi, qi, nb_rxd, &hairpin_conf);
2388                 i++;
2389                 if (diag == 0)
2390                         continue;
2391
2392                 /* Fail to setup rx queue, return */
2393                 if (rte_atomic16_cmpset(&(port->port_status),
2394                                         RTE_PORT_HANDLING,
2395                                         RTE_PORT_STOPPED) == 0)
2396                         printf("Port %d can not be set back "
2397                                         "to stopped\n", pi);
2398                 printf("Fail to configure port %d hairpin "
2399                                 "queues\n", pi);
2400                 /* try to reconfigure queues next time */
2401                 port->need_reconfig_queues = 1;
2402                 return -1;
2403         }
2404         return 0;
2405 }
2406
2407 /* Configure the Rx with optional split. */
2408 int
2409 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2410                uint16_t nb_rx_desc, unsigned int socket_id,
2411                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2412 {
2413         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2414         unsigned int i, mp_n;
2415         int ret;
2416
2417         if (rx_pkt_nb_segs <= 1 ||
2418             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2419                 rx_conf->rx_seg = NULL;
2420                 rx_conf->rx_nseg = 0;
2421                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2422                                              nb_rx_desc, socket_id,
2423                                              rx_conf, mp);
2424                 return ret;
2425         }
2426         for (i = 0; i < rx_pkt_nb_segs; i++) {
2427                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2428                 struct rte_mempool *mpx;
2429                 /*
2430                  * Use last valid pool for the segments with number
2431                  * exceeding the pool index.
2432                  */
2433                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2434                 mpx = mbuf_pool_find(socket_id, mp_n);
2435                 /* Handle zero as mbuf data buffer size. */
2436                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2437                                    rx_pkt_seg_lengths[i] :
2438                                    mbuf_data_size[mp_n];
2439                 rx_seg->offset = i < rx_pkt_nb_offs ?
2440                                    rx_pkt_seg_offsets[i] : 0;
2441                 rx_seg->mp = mpx ? mpx : mp;
2442         }
2443         rx_conf->rx_nseg = rx_pkt_nb_segs;
2444         rx_conf->rx_seg = rx_useg;
2445         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2446                                     socket_id, rx_conf, NULL);
2447         rx_conf->rx_seg = NULL;
2448         rx_conf->rx_nseg = 0;
2449         return ret;
2450 }
2451
2452 int
2453 start_port(portid_t pid)
2454 {
2455         int diag, need_check_link_status = -1;
2456         portid_t pi;
2457         portid_t p_pi = RTE_MAX_ETHPORTS;
2458         portid_t pl[RTE_MAX_ETHPORTS];
2459         portid_t peer_pl[RTE_MAX_ETHPORTS];
2460         uint16_t cnt_pi = 0;
2461         uint16_t cfg_pi = 0;
2462         int peer_pi;
2463         queueid_t qi;
2464         struct rte_port *port;
2465         struct rte_ether_addr mac_addr;
2466         struct rte_eth_hairpin_cap cap;
2467
2468         if (port_id_is_invalid(pid, ENABLED_WARN))
2469                 return 0;
2470
2471         if(dcb_config)
2472                 dcb_test = 1;
2473         RTE_ETH_FOREACH_DEV(pi) {
2474                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2475                         continue;
2476
2477                 need_check_link_status = 0;
2478                 port = &ports[pi];
2479                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2480                                                  RTE_PORT_HANDLING) == 0) {
2481                         printf("Port %d is now not stopped\n", pi);
2482                         continue;
2483                 }
2484
2485                 if (port->need_reconfig > 0) {
2486                         port->need_reconfig = 0;
2487
2488                         if (flow_isolate_all) {
2489                                 int ret = port_flow_isolate(pi, 1);
2490                                 if (ret) {
2491                                         printf("Failed to apply isolated"
2492                                                " mode on port %d\n", pi);
2493                                         return -1;
2494                                 }
2495                         }
2496                         configure_rxtx_dump_callbacks(0);
2497                         printf("Configuring Port %d (socket %u)\n", pi,
2498                                         port->socket_id);
2499                         if (nb_hairpinq > 0 &&
2500                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2501                                 printf("Port %d doesn't support hairpin "
2502                                        "queues\n", pi);
2503                                 return -1;
2504                         }
2505                         /* configure port */
2506                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2507                                                      nb_txq + nb_hairpinq,
2508                                                      &(port->dev_conf));
2509                         if (diag != 0) {
2510                                 if (rte_atomic16_cmpset(&(port->port_status),
2511                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2512                                         printf("Port %d can not be set back "
2513                                                         "to stopped\n", pi);
2514                                 printf("Fail to configure port %d\n", pi);
2515                                 /* try to reconfigure port next time */
2516                                 port->need_reconfig = 1;
2517                                 return -1;
2518                         }
2519                 }
2520                 if (port->need_reconfig_queues > 0) {
2521                         port->need_reconfig_queues = 0;
2522                         /* setup tx queues */
2523                         for (qi = 0; qi < nb_txq; qi++) {
2524                                 if ((numa_support) &&
2525                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2526                                         diag = rte_eth_tx_queue_setup(pi, qi,
2527                                                 port->nb_tx_desc[qi],
2528                                                 txring_numa[pi],
2529                                                 &(port->tx_conf[qi]));
2530                                 else
2531                                         diag = rte_eth_tx_queue_setup(pi, qi,
2532                                                 port->nb_tx_desc[qi],
2533                                                 port->socket_id,
2534                                                 &(port->tx_conf[qi]));
2535
2536                                 if (diag == 0)
2537                                         continue;
2538
2539                                 /* Fail to setup tx queue, return */
2540                                 if (rte_atomic16_cmpset(&(port->port_status),
2541                                                         RTE_PORT_HANDLING,
2542                                                         RTE_PORT_STOPPED) == 0)
2543                                         printf("Port %d can not be set back "
2544                                                         "to stopped\n", pi);
2545                                 printf("Fail to configure port %d tx queues\n",
2546                                        pi);
2547                                 /* try to reconfigure queues next time */
2548                                 port->need_reconfig_queues = 1;
2549                                 return -1;
2550                         }
2551                         for (qi = 0; qi < nb_rxq; qi++) {
2552                                 /* setup rx queues */
2553                                 if ((numa_support) &&
2554                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2555                                         struct rte_mempool * mp =
2556                                                 mbuf_pool_find
2557                                                         (rxring_numa[pi], 0);
2558                                         if (mp == NULL) {
2559                                                 printf("Failed to setup RX queue:"
2560                                                         "No mempool allocation"
2561                                                         " on the socket %d\n",
2562                                                         rxring_numa[pi]);
2563                                                 return -1;
2564                                         }
2565
2566                                         diag = rx_queue_setup(pi, qi,
2567                                              port->nb_rx_desc[qi],
2568                                              rxring_numa[pi],
2569                                              &(port->rx_conf[qi]),
2570                                              mp);
2571                                 } else {
2572                                         struct rte_mempool *mp =
2573                                                 mbuf_pool_find
2574                                                         (port->socket_id, 0);
2575                                         if (mp == NULL) {
2576                                                 printf("Failed to setup RX queue:"
2577                                                         "No mempool allocation"
2578                                                         " on the socket %d\n",
2579                                                         port->socket_id);
2580                                                 return -1;
2581                                         }
2582                                         diag = rx_queue_setup(pi, qi,
2583                                              port->nb_rx_desc[qi],
2584                                              port->socket_id,
2585                                              &(port->rx_conf[qi]),
2586                                              mp);
2587                                 }
2588                                 if (diag == 0)
2589                                         continue;
2590
2591                                 /* Fail to setup rx queue, return */
2592                                 if (rte_atomic16_cmpset(&(port->port_status),
2593                                                         RTE_PORT_HANDLING,
2594                                                         RTE_PORT_STOPPED) == 0)
2595                                         printf("Port %d can not be set back "
2596                                                         "to stopped\n", pi);
2597                                 printf("Fail to configure port %d rx queues\n",
2598                                        pi);
2599                                 /* try to reconfigure queues next time */
2600                                 port->need_reconfig_queues = 1;
2601                                 return -1;
2602                         }
2603                         /* setup hairpin queues */
2604                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2605                                 return -1;
2606                 }
2607                 configure_rxtx_dump_callbacks(verbose_level);
2608                 if (clear_ptypes) {
2609                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2610                                         NULL, 0);
2611                         if (diag < 0)
2612                                 printf(
2613                                 "Port %d: Failed to disable Ptype parsing\n",
2614                                 pi);
2615                 }
2616
2617                 p_pi = pi;
2618                 cnt_pi++;
2619
2620                 /* start port */
2621                 if (rte_eth_dev_start(pi) < 0) {
2622                         printf("Fail to start port %d\n", pi);
2623
2624                         /* Fail to setup rx queue, return */
2625                         if (rte_atomic16_cmpset(&(port->port_status),
2626                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2627                                 printf("Port %d can not be set back to "
2628                                                         "stopped\n", pi);
2629                         continue;
2630                 }
2631
2632                 if (rte_atomic16_cmpset(&(port->port_status),
2633                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2634                         printf("Port %d can not be set into started\n", pi);
2635
2636                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2637                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2638                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2639                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2640                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2641
2642                 /* at least one port started, need checking link status */
2643                 need_check_link_status = 1;
2644
2645                 pl[cfg_pi++] = pi;
2646         }
2647
2648         if (need_check_link_status == 1 && !no_link_check)
2649                 check_all_ports_link_status(RTE_PORT_ALL);
2650         else if (need_check_link_status == 0)
2651                 printf("Please stop the ports first\n");
2652
2653         if (hairpin_mode & 0xf) {
2654                 uint16_t i;
2655                 int j;
2656
2657                 /* bind all started hairpin ports */
2658                 for (i = 0; i < cfg_pi; i++) {
2659                         pi = pl[i];
2660                         /* bind current Tx to all peer Rx */
2661                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2662                                                         RTE_MAX_ETHPORTS, 1);
2663                         if (peer_pi < 0)
2664                                 return peer_pi;
2665                         for (j = 0; j < peer_pi; j++) {
2666                                 if (!port_is_started(peer_pl[j]))
2667                                         continue;
2668                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2669                                 if (diag < 0) {
2670                                         printf("Error during binding hairpin"
2671                                                " Tx port %u to %u: %s\n",
2672                                                pi, peer_pl[j],
2673                                                rte_strerror(-diag));
2674                                         return -1;
2675                                 }
2676                         }
2677                         /* bind all peer Tx to current Rx */
2678                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2679                                                         RTE_MAX_ETHPORTS, 0);
2680                         if (peer_pi < 0)
2681                                 return peer_pi;
2682                         for (j = 0; j < peer_pi; j++) {
2683                                 if (!port_is_started(peer_pl[j]))
2684                                         continue;
2685                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2686                                 if (diag < 0) {
2687                                         printf("Error during binding hairpin"
2688                                                " Tx port %u to %u: %s\n",
2689                                                peer_pl[j], pi,
2690                                                rte_strerror(-diag));
2691                                         return -1;
2692                                 }
2693                         }
2694                 }
2695         }
2696
2697         printf("Done\n");
2698         return 0;
2699 }
2700
2701 void
2702 stop_port(portid_t pid)
2703 {
2704         portid_t pi;
2705         struct rte_port *port;
2706         int need_check_link_status = 0;
2707         portid_t peer_pl[RTE_MAX_ETHPORTS];
2708         int peer_pi;
2709
2710         if (dcb_test) {
2711                 dcb_test = 0;
2712                 dcb_config = 0;
2713         }
2714
2715         if (port_id_is_invalid(pid, ENABLED_WARN))
2716                 return;
2717
2718         printf("Stopping ports...\n");
2719
2720         RTE_ETH_FOREACH_DEV(pi) {
2721                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2722                         continue;
2723
2724                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2725                         printf("Please remove port %d from forwarding configuration.\n", pi);
2726                         continue;
2727                 }
2728
2729                 if (port_is_bonding_slave(pi)) {
2730                         printf("Please remove port %d from bonded device.\n", pi);
2731                         continue;
2732                 }
2733
2734                 port = &ports[pi];
2735                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2736                                                 RTE_PORT_HANDLING) == 0)
2737                         continue;
2738
2739                 if (hairpin_mode & 0xf) {
2740                         int j;
2741
2742                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2743                         /* unbind all peer Tx from current Rx */
2744                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2745                                                         RTE_MAX_ETHPORTS, 0);
2746                         if (peer_pi < 0)
2747                                 continue;
2748                         for (j = 0; j < peer_pi; j++) {
2749                                 if (!port_is_started(peer_pl[j]))
2750                                         continue;
2751                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2752                         }
2753                 }
2754
2755                 if (port->flow_list)
2756                         port_flow_flush(pi);
2757
2758                 if (rte_eth_dev_stop(pi) != 0)
2759                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2760                                 pi);
2761
2762                 if (rte_atomic16_cmpset(&(port->port_status),
2763                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2764                         printf("Port %d can not be set into stopped\n", pi);
2765                 need_check_link_status = 1;
2766         }
2767         if (need_check_link_status && !no_link_check)
2768                 check_all_ports_link_status(RTE_PORT_ALL);
2769
2770         printf("Done\n");
2771 }
2772
2773 static void
2774 remove_invalid_ports_in(portid_t *array, portid_t *total)
2775 {
2776         portid_t i;
2777         portid_t new_total = 0;
2778
2779         for (i = 0; i < *total; i++)
2780                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2781                         array[new_total] = array[i];
2782                         new_total++;
2783                 }
2784         *total = new_total;
2785 }
2786
2787 static void
2788 remove_invalid_ports(void)
2789 {
2790         remove_invalid_ports_in(ports_ids, &nb_ports);
2791         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2792         nb_cfg_ports = nb_fwd_ports;
2793 }
2794
2795 void
2796 close_port(portid_t pid)
2797 {
2798         portid_t pi;
2799         struct rte_port *port;
2800
2801         if (port_id_is_invalid(pid, ENABLED_WARN))
2802                 return;
2803
2804         printf("Closing ports...\n");
2805
2806         RTE_ETH_FOREACH_DEV(pi) {
2807                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2808                         continue;
2809
2810                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2811                         printf("Please remove port %d from forwarding configuration.\n", pi);
2812                         continue;
2813                 }
2814
2815                 if (port_is_bonding_slave(pi)) {
2816                         printf("Please remove port %d from bonded device.\n", pi);
2817                         continue;
2818                 }
2819
2820                 port = &ports[pi];
2821                 if (rte_atomic16_cmpset(&(port->port_status),
2822                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2823                         printf("Port %d is already closed\n", pi);
2824                         continue;
2825                 }
2826
2827                 port_flow_flush(pi);
2828                 rte_eth_dev_close(pi);
2829         }
2830
2831         remove_invalid_ports();
2832         printf("Done\n");
2833 }
2834
2835 void
2836 reset_port(portid_t pid)
2837 {
2838         int diag;
2839         portid_t pi;
2840         struct rte_port *port;
2841
2842         if (port_id_is_invalid(pid, ENABLED_WARN))
2843                 return;
2844
2845         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2846                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2847                 printf("Can not reset port(s), please stop port(s) first.\n");
2848                 return;
2849         }
2850
2851         printf("Resetting ports...\n");
2852
2853         RTE_ETH_FOREACH_DEV(pi) {
2854                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2855                         continue;
2856
2857                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2858                         printf("Please remove port %d from forwarding "
2859                                "configuration.\n", pi);
2860                         continue;
2861                 }
2862
2863                 if (port_is_bonding_slave(pi)) {
2864                         printf("Please remove port %d from bonded device.\n",
2865                                pi);
2866                         continue;
2867                 }
2868
2869                 diag = rte_eth_dev_reset(pi);
2870                 if (diag == 0) {
2871                         port = &ports[pi];
2872                         port->need_reconfig = 1;
2873                         port->need_reconfig_queues = 1;
2874                 } else {
2875                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2876                 }
2877         }
2878
2879         printf("Done\n");
2880 }
2881
2882 void
2883 attach_port(char *identifier)
2884 {
2885         portid_t pi;
2886         struct rte_dev_iterator iterator;
2887
2888         printf("Attaching a new port...\n");
2889
2890         if (identifier == NULL) {
2891                 printf("Invalid parameters are specified\n");
2892                 return;
2893         }
2894
2895         if (rte_dev_probe(identifier) < 0) {
2896                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2897                 return;
2898         }
2899
2900         /* first attach mode: event */
2901         if (setup_on_probe_event) {
2902                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2903                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2904                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2905                                         ports[pi].need_setup != 0)
2906                                 setup_attached_port(pi);
2907                 return;
2908         }
2909
2910         /* second attach mode: iterator */
2911         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2912                 /* setup ports matching the devargs used for probing */
2913                 if (port_is_forwarding(pi))
2914                         continue; /* port was already attached before */
2915                 setup_attached_port(pi);
2916         }
2917 }
2918
2919 static void
2920 setup_attached_port(portid_t pi)
2921 {
2922         unsigned int socket_id;
2923         int ret;
2924
2925         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2926         /* if socket_id is invalid, set to the first available socket. */
2927         if (check_socket_id(socket_id) < 0)
2928                 socket_id = socket_ids[0];
2929         reconfig(pi, socket_id);
2930         ret = rte_eth_promiscuous_enable(pi);
2931         if (ret != 0)
2932                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2933                         pi, rte_strerror(-ret));
2934
2935         ports_ids[nb_ports++] = pi;
2936         fwd_ports_ids[nb_fwd_ports++] = pi;
2937         nb_cfg_ports = nb_fwd_ports;
2938         ports[pi].need_setup = 0;
2939         ports[pi].port_status = RTE_PORT_STOPPED;
2940
2941         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2942         printf("Done\n");
2943 }
2944
2945 static void
2946 detach_device(struct rte_device *dev)
2947 {
2948         portid_t sibling;
2949
2950         if (dev == NULL) {
2951                 printf("Device already removed\n");
2952                 return;
2953         }
2954
2955         printf("Removing a device...\n");
2956
2957         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2958                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2959                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2960                                 printf("Port %u not stopped\n", sibling);
2961                                 return;
2962                         }
2963                         port_flow_flush(sibling);
2964                 }
2965         }
2966
2967         if (rte_dev_remove(dev) < 0) {
2968                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2969                 return;
2970         }
2971         remove_invalid_ports();
2972
2973         printf("Device is detached\n");
2974         printf("Now total ports is %d\n", nb_ports);
2975         printf("Done\n");
2976         return;
2977 }
2978
2979 void
2980 detach_port_device(portid_t port_id)
2981 {
2982         if (port_id_is_invalid(port_id, ENABLED_WARN))
2983                 return;
2984
2985         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2986                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2987                         printf("Port not stopped\n");
2988                         return;
2989                 }
2990                 printf("Port was not closed\n");
2991         }
2992
2993         detach_device(rte_eth_devices[port_id].device);
2994 }
2995
2996 void
2997 detach_devargs(char *identifier)
2998 {
2999         struct rte_dev_iterator iterator;
3000         struct rte_devargs da;
3001         portid_t port_id;
3002
3003         printf("Removing a device...\n");
3004
3005         memset(&da, 0, sizeof(da));
3006         if (rte_devargs_parsef(&da, "%s", identifier)) {
3007                 printf("cannot parse identifier\n");
3008                 if (da.args)
3009                         free(da.args);
3010                 return;
3011         }
3012
3013         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3014                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3015                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3016                                 printf("Port %u not stopped\n", port_id);
3017                                 rte_eth_iterator_cleanup(&iterator);
3018                                 return;
3019                         }
3020                         port_flow_flush(port_id);
3021                 }
3022         }
3023
3024         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3025                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3026                             da.name, da.bus->name);
3027                 return;
3028         }
3029
3030         remove_invalid_ports();
3031
3032         printf("Device %s is detached\n", identifier);
3033         printf("Now total ports is %d\n", nb_ports);
3034         printf("Done\n");
3035 }
3036
3037 void
3038 pmd_test_exit(void)
3039 {
3040         portid_t pt_id;
3041         unsigned int i;
3042         int ret;
3043
3044         if (test_done == 0)
3045                 stop_packet_forwarding();
3046
3047         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3048                 if (mempools[i]) {
3049                         if (mp_alloc_type == MP_ALLOC_ANON)
3050                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3051                                                      NULL);
3052                 }
3053         }
3054         if (ports != NULL) {
3055                 no_link_check = 1;
3056                 RTE_ETH_FOREACH_DEV(pt_id) {
3057                         printf("\nStopping port %d...\n", pt_id);
3058                         fflush(stdout);
3059                         stop_port(pt_id);
3060                 }
3061                 RTE_ETH_FOREACH_DEV(pt_id) {
3062                         printf("\nShutting down port %d...\n", pt_id);
3063                         fflush(stdout);
3064                         close_port(pt_id);
3065                 }
3066         }
3067
3068         if (hot_plug) {
3069                 ret = rte_dev_event_monitor_stop();
3070                 if (ret) {
3071                         RTE_LOG(ERR, EAL,
3072                                 "fail to stop device event monitor.");
3073                         return;
3074                 }
3075
3076                 ret = rte_dev_event_callback_unregister(NULL,
3077                         dev_event_callback, NULL);
3078                 if (ret < 0) {
3079                         RTE_LOG(ERR, EAL,
3080                                 "fail to unregister device event callback.\n");
3081                         return;
3082                 }
3083
3084                 ret = rte_dev_hotplug_handle_disable();
3085                 if (ret) {
3086                         RTE_LOG(ERR, EAL,
3087                                 "fail to disable hotplug handling.\n");
3088                         return;
3089                 }
3090         }
3091         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3092                 if (mempools[i])
3093                         rte_mempool_free(mempools[i]);
3094         }
3095
3096         printf("\nBye...\n");
3097 }
3098
3099 typedef void (*cmd_func_t)(void);
3100 struct pmd_test_command {
3101         const char *cmd_name;
3102         cmd_func_t cmd_func;
3103 };
3104
3105 /* Check the link status of all ports in up to 9s, and print them finally */
3106 static void
3107 check_all_ports_link_status(uint32_t port_mask)
3108 {
3109 #define CHECK_INTERVAL 100 /* 100ms */
3110 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3111         portid_t portid;
3112         uint8_t count, all_ports_up, print_flag = 0;
3113         struct rte_eth_link link;
3114         int ret;
3115         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3116
3117         printf("Checking link statuses...\n");
3118         fflush(stdout);
3119         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3120                 all_ports_up = 1;
3121                 RTE_ETH_FOREACH_DEV(portid) {
3122                         if ((port_mask & (1 << portid)) == 0)
3123                                 continue;
3124                         memset(&link, 0, sizeof(link));
3125                         ret = rte_eth_link_get_nowait(portid, &link);
3126                         if (ret < 0) {
3127                                 all_ports_up = 0;
3128                                 if (print_flag == 1)
3129                                         printf("Port %u link get failed: %s\n",
3130                                                 portid, rte_strerror(-ret));
3131                                 continue;
3132                         }
3133                         /* print link status if flag set */
3134                         if (print_flag == 1) {
3135                                 rte_eth_link_to_str(link_status,
3136                                         sizeof(link_status), &link);
3137                                 printf("Port %d %s\n", portid, link_status);
3138                                 continue;
3139                         }
3140                         /* clear all_ports_up flag if any link down */
3141                         if (link.link_status == ETH_LINK_DOWN) {
3142                                 all_ports_up = 0;
3143                                 break;
3144                         }
3145                 }
3146                 /* after finally printing all link status, get out */
3147                 if (print_flag == 1)
3148                         break;
3149
3150                 if (all_ports_up == 0) {
3151                         fflush(stdout);
3152                         rte_delay_ms(CHECK_INTERVAL);
3153                 }
3154
3155                 /* set the print_flag if all ports up or timeout */
3156                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3157                         print_flag = 1;
3158                 }
3159
3160                 if (lsc_interrupt)
3161                         break;
3162         }
3163 }
3164
3165 static void
3166 rmv_port_callback(void *arg)
3167 {
3168         int need_to_start = 0;
3169         int org_no_link_check = no_link_check;
3170         portid_t port_id = (intptr_t)arg;
3171         struct rte_device *dev;
3172
3173         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3174
3175         if (!test_done && port_is_forwarding(port_id)) {
3176                 need_to_start = 1;
3177                 stop_packet_forwarding();
3178         }
3179         no_link_check = 1;
3180         stop_port(port_id);
3181         no_link_check = org_no_link_check;
3182
3183         /* Save rte_device pointer before closing ethdev port */
3184         dev = rte_eth_devices[port_id].device;
3185         close_port(port_id);
3186         detach_device(dev); /* might be already removed or have more ports */
3187
3188         if (need_to_start)
3189                 start_packet_forwarding(0);
3190 }
3191
3192 /* This function is used by the interrupt thread */
3193 static int
3194 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3195                   void *ret_param)
3196 {
3197         RTE_SET_USED(param);
3198         RTE_SET_USED(ret_param);
3199
3200         if (type >= RTE_ETH_EVENT_MAX) {
3201                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3202                         port_id, __func__, type);
3203                 fflush(stderr);
3204         } else if (event_print_mask & (UINT32_C(1) << type)) {
3205                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3206                         eth_event_desc[type]);
3207                 fflush(stdout);
3208         }
3209
3210         switch (type) {
3211         case RTE_ETH_EVENT_NEW:
3212                 ports[port_id].need_setup = 1;
3213                 ports[port_id].port_status = RTE_PORT_HANDLING;
3214                 break;
3215         case RTE_ETH_EVENT_INTR_RMV:
3216                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3217                         break;
3218                 if (rte_eal_alarm_set(100000,
3219                                 rmv_port_callback, (void *)(intptr_t)port_id))
3220                         fprintf(stderr, "Could not set up deferred device removal\n");
3221                 break;
3222         case RTE_ETH_EVENT_DESTROY:
3223                 ports[port_id].port_status = RTE_PORT_CLOSED;
3224                 printf("Port %u is closed\n", port_id);
3225                 break;
3226         default:
3227                 break;
3228         }
3229         return 0;
3230 }
3231
3232 static int
3233 register_eth_event_callback(void)
3234 {
3235         int ret;
3236         enum rte_eth_event_type event;
3237
3238         for (event = RTE_ETH_EVENT_UNKNOWN;
3239                         event < RTE_ETH_EVENT_MAX; event++) {
3240                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3241                                 event,
3242                                 eth_event_callback,
3243                                 NULL);
3244                 if (ret != 0) {
3245                         TESTPMD_LOG(ERR, "Failed to register callback for "
3246                                         "%s event\n", eth_event_desc[event]);
3247                         return -1;
3248                 }
3249         }
3250
3251         return 0;
3252 }
3253
3254 /* This function is used by the interrupt thread */
3255 static void
3256 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3257                              __rte_unused void *arg)
3258 {
3259         uint16_t port_id;
3260         int ret;
3261
3262         if (type >= RTE_DEV_EVENT_MAX) {
3263                 fprintf(stderr, "%s called upon invalid event %d\n",
3264                         __func__, type);
3265                 fflush(stderr);
3266         }
3267
3268         switch (type) {
3269         case RTE_DEV_EVENT_REMOVE:
3270                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3271                         device_name);
3272                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3273                 if (ret) {
3274                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3275                                 device_name);
3276                         return;
3277                 }
3278                 /*
3279                  * Because the user's callback is invoked in eal interrupt
3280                  * callback, the interrupt callback need to be finished before
3281                  * it can be unregistered when detaching device. So finish
3282                  * callback soon and use a deferred removal to detach device
3283                  * is need. It is a workaround, once the device detaching be
3284                  * moved into the eal in the future, the deferred removal could
3285                  * be deleted.
3286                  */
3287                 if (rte_eal_alarm_set(100000,
3288                                 rmv_port_callback, (void *)(intptr_t)port_id))
3289                         RTE_LOG(ERR, EAL,
3290                                 "Could not set up deferred device removal\n");
3291                 break;
3292         case RTE_DEV_EVENT_ADD:
3293                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3294                         device_name);
3295                 /* TODO: After finish kernel driver binding,
3296                  * begin to attach port.
3297                  */
3298                 break;
3299         default:
3300                 break;
3301         }
3302 }
3303
3304 static void
3305 rxtx_port_config(struct rte_port *port)
3306 {
3307         uint16_t qid;
3308         uint64_t offloads;
3309
3310         for (qid = 0; qid < nb_rxq; qid++) {
3311                 offloads = port->rx_conf[qid].offloads;
3312                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3313                 if (offloads != 0)
3314                         port->rx_conf[qid].offloads = offloads;
3315
3316                 /* Check if any Rx parameters have been passed */
3317                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3318                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3319
3320                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3321                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3322
3323                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3324                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3325
3326                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3327                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3328
3329                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3330                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3331
3332                 port->nb_rx_desc[qid] = nb_rxd;
3333         }
3334
3335         for (qid = 0; qid < nb_txq; qid++) {
3336                 offloads = port->tx_conf[qid].offloads;
3337                 port->tx_conf[qid] = port->dev_info.default_txconf;
3338                 if (offloads != 0)
3339                         port->tx_conf[qid].offloads = offloads;
3340
3341                 /* Check if any Tx parameters have been passed */
3342                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3343                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3344
3345                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3346                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3347
3348                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3349                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3350
3351                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3352                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3353
3354                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3355                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3356
3357                 port->nb_tx_desc[qid] = nb_txd;
3358         }
3359 }
3360
3361 /*
3362  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3363  * MTU is also aligned if JUMBO_FRAME offload is not set.
3364  *
3365  * port->dev_info should be set before calling this function.
3366  *
3367  * return 0 on success, negative on error
3368  */
3369 int
3370 update_jumbo_frame_offload(portid_t portid)
3371 {
3372         struct rte_port *port = &ports[portid];
3373         uint32_t eth_overhead;
3374         uint64_t rx_offloads;
3375         int ret;
3376         bool on;
3377
3378         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3379         if (port->dev_info.max_mtu != UINT16_MAX &&
3380             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3381                 eth_overhead = port->dev_info.max_rx_pktlen -
3382                                 port->dev_info.max_mtu;
3383         else
3384                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3385
3386         rx_offloads = port->dev_conf.rxmode.offloads;
3387
3388         /* Default config value is 0 to use PMD specific overhead */
3389         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3390                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3391
3392         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3393                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3394                 on = false;
3395         } else {
3396                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3397                         printf("Frame size (%u) is not supported by port %u\n",
3398                                 port->dev_conf.rxmode.max_rx_pkt_len,
3399                                 portid);
3400                         return -1;
3401                 }
3402                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3403                 on = true;
3404         }
3405
3406         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3407                 uint16_t qid;
3408
3409                 port->dev_conf.rxmode.offloads = rx_offloads;
3410
3411                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3412                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3413                         if (on)
3414                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3415                         else
3416                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3417                 }
3418         }
3419
3420         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3421          * if unset do it here
3422          */
3423         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3424                 ret = rte_eth_dev_set_mtu(portid,
3425                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3426                 if (ret)
3427                         printf("Failed to set MTU to %u for port %u\n",
3428                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3429                                 portid);
3430         }
3431
3432         return 0;
3433 }
3434
3435 void
3436 init_port_config(void)
3437 {
3438         portid_t pid;
3439         struct rte_port *port;
3440         int ret;
3441
3442         RTE_ETH_FOREACH_DEV(pid) {
3443                 port = &ports[pid];
3444                 port->dev_conf.fdir_conf = fdir_conf;
3445
3446                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3447                 if (ret != 0)
3448                         return;
3449
3450                 if (nb_rxq > 1) {
3451                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3452                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3453                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3454                 } else {
3455                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3456                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3457                 }
3458
3459                 if (port->dcb_flag == 0) {
3460                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3461                                 port->dev_conf.rxmode.mq_mode =
3462                                         (enum rte_eth_rx_mq_mode)
3463                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3464                         else
3465                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3466                 }
3467
3468                 rxtx_port_config(port);
3469
3470                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3471                 if (ret != 0)
3472                         return;
3473
3474 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3475                 rte_pmd_ixgbe_bypass_init(pid);
3476 #endif
3477
3478                 if (lsc_interrupt &&
3479                     (rte_eth_devices[pid].data->dev_flags &
3480                      RTE_ETH_DEV_INTR_LSC))
3481                         port->dev_conf.intr_conf.lsc = 1;
3482                 if (rmv_interrupt &&
3483                     (rte_eth_devices[pid].data->dev_flags &
3484                      RTE_ETH_DEV_INTR_RMV))
3485                         port->dev_conf.intr_conf.rmv = 1;
3486         }
3487 }
3488
3489 void set_port_slave_flag(portid_t slave_pid)
3490 {
3491         struct rte_port *port;
3492
3493         port = &ports[slave_pid];
3494         port->slave_flag = 1;
3495 }
3496
3497 void clear_port_slave_flag(portid_t slave_pid)
3498 {
3499         struct rte_port *port;
3500
3501         port = &ports[slave_pid];
3502         port->slave_flag = 0;
3503 }
3504
3505 uint8_t port_is_bonding_slave(portid_t slave_pid)
3506 {
3507         struct rte_port *port;
3508
3509         port = &ports[slave_pid];
3510         if ((rte_eth_devices[slave_pid].data->dev_flags &
3511             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3512                 return 1;
3513         return 0;
3514 }
3515
3516 const uint16_t vlan_tags[] = {
3517                 0,  1,  2,  3,  4,  5,  6,  7,
3518                 8,  9, 10, 11,  12, 13, 14, 15,
3519                 16, 17, 18, 19, 20, 21, 22, 23,
3520                 24, 25, 26, 27, 28, 29, 30, 31
3521 };
3522
3523 static  int
3524 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3525                  enum dcb_mode_enable dcb_mode,
3526                  enum rte_eth_nb_tcs num_tcs,
3527                  uint8_t pfc_en)
3528 {
3529         uint8_t i;
3530         int32_t rc;
3531         struct rte_eth_rss_conf rss_conf;
3532
3533         /*
3534          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3535          * given above, and the number of traffic classes available for use.
3536          */
3537         if (dcb_mode == DCB_VT_ENABLED) {
3538                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3539                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3540                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3541                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3542
3543                 /* VMDQ+DCB RX and TX configurations */
3544                 vmdq_rx_conf->enable_default_pool = 0;
3545                 vmdq_rx_conf->default_pool = 0;
3546                 vmdq_rx_conf->nb_queue_pools =
3547                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3548                 vmdq_tx_conf->nb_queue_pools =
3549                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3550
3551                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3552                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3553                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3554                         vmdq_rx_conf->pool_map[i].pools =
3555                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3556                 }
3557                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3558                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3559                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3560                 }
3561
3562                 /* set DCB mode of RX and TX of multiple queues */
3563                 eth_conf->rxmode.mq_mode =
3564                                 (enum rte_eth_rx_mq_mode)
3565                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3566                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3567         } else {
3568                 struct rte_eth_dcb_rx_conf *rx_conf =
3569                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3570                 struct rte_eth_dcb_tx_conf *tx_conf =
3571                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3572
3573                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3574
3575                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3576                 if (rc != 0)
3577                         return rc;
3578
3579                 rx_conf->nb_tcs = num_tcs;
3580                 tx_conf->nb_tcs = num_tcs;
3581
3582                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3583                         rx_conf->dcb_tc[i] = i % num_tcs;
3584                         tx_conf->dcb_tc[i] = i % num_tcs;
3585                 }
3586
3587                 eth_conf->rxmode.mq_mode =
3588                                 (enum rte_eth_rx_mq_mode)
3589                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3590                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3591                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3592         }
3593
3594         if (pfc_en)
3595                 eth_conf->dcb_capability_en =
3596                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3597         else
3598                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3599
3600         return 0;
3601 }
3602
3603 int
3604 init_port_dcb_config(portid_t pid,
3605                      enum dcb_mode_enable dcb_mode,
3606                      enum rte_eth_nb_tcs num_tcs,
3607                      uint8_t pfc_en)
3608 {
3609         struct rte_eth_conf port_conf;
3610         struct rte_port *rte_port;
3611         int retval;
3612         uint16_t i;
3613
3614         rte_port = &ports[pid];
3615
3616         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3617         /* Enter DCB configuration status */
3618         dcb_config = 1;
3619
3620         port_conf.rxmode = rte_port->dev_conf.rxmode;
3621         port_conf.txmode = rte_port->dev_conf.txmode;
3622
3623         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3624         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3625         if (retval < 0)
3626                 return retval;
3627         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3628
3629         /* re-configure the device . */
3630         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3631         if (retval < 0)
3632                 return retval;
3633
3634         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3635         if (retval != 0)
3636                 return retval;
3637
3638         /* If dev_info.vmdq_pool_base is greater than 0,
3639          * the queue id of vmdq pools is started after pf queues.
3640          */
3641         if (dcb_mode == DCB_VT_ENABLED &&
3642             rte_port->dev_info.vmdq_pool_base > 0) {
3643                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3644                         " for port %d.", pid);
3645                 return -1;
3646         }
3647
3648         /* Assume the ports in testpmd have the same dcb capability
3649          * and has the same number of rxq and txq in dcb mode
3650          */
3651         if (dcb_mode == DCB_VT_ENABLED) {
3652                 if (rte_port->dev_info.max_vfs > 0) {
3653                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3654                         nb_txq = rte_port->dev_info.nb_tx_queues;
3655                 } else {
3656                         nb_rxq = rte_port->dev_info.max_rx_queues;
3657                         nb_txq = rte_port->dev_info.max_tx_queues;
3658                 }
3659         } else {
3660                 /*if vt is disabled, use all pf queues */
3661                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3662                         nb_rxq = rte_port->dev_info.max_rx_queues;
3663                         nb_txq = rte_port->dev_info.max_tx_queues;
3664                 } else {
3665                         nb_rxq = (queueid_t)num_tcs;
3666                         nb_txq = (queueid_t)num_tcs;
3667
3668                 }
3669         }
3670         rx_free_thresh = 64;
3671
3672         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3673
3674         rxtx_port_config(rte_port);
3675         /* VLAN filter */
3676         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3677         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3678                 rx_vft_set(pid, vlan_tags[i], 1);
3679
3680         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3681         if (retval != 0)
3682                 return retval;
3683
3684         rte_port->dcb_flag = 1;
3685
3686         return 0;
3687 }
3688
3689 static void
3690 init_port(void)
3691 {
3692         int i;
3693
3694         /* Configuration of Ethernet ports. */
3695         ports = rte_zmalloc("testpmd: ports",
3696                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3697                             RTE_CACHE_LINE_SIZE);
3698         if (ports == NULL) {
3699                 rte_exit(EXIT_FAILURE,
3700                                 "rte_zmalloc(%d struct rte_port) failed\n",
3701                                 RTE_MAX_ETHPORTS);
3702         }
3703         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3704                 LIST_INIT(&ports[i].flow_tunnel_list);
3705         /* Initialize ports NUMA structures */
3706         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3707         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3708         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3709 }
3710
3711 static void
3712 force_quit(void)
3713 {
3714         pmd_test_exit();
3715         prompt_exit();
3716 }
3717
3718 static void
3719 print_stats(void)
3720 {
3721         uint8_t i;
3722         const char clr[] = { 27, '[', '2', 'J', '\0' };
3723         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3724
3725         /* Clear screen and move to top left */
3726         printf("%s%s", clr, top_left);
3727
3728         printf("\nPort statistics ====================================");
3729         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3730                 nic_stats_display(fwd_ports_ids[i]);
3731
3732         fflush(stdout);
3733 }
3734
3735 static void
3736 signal_handler(int signum)
3737 {
3738         if (signum == SIGINT || signum == SIGTERM) {
3739                 printf("\nSignal %d received, preparing to exit...\n",
3740                                 signum);
3741 #ifdef RTE_LIB_PDUMP
3742                 /* uninitialize packet capture framework */
3743                 rte_pdump_uninit();
3744 #endif
3745 #ifdef RTE_LIB_LATENCYSTATS
3746                 if (latencystats_enabled != 0)
3747                         rte_latencystats_uninit();
3748 #endif
3749                 force_quit();
3750                 /* Set flag to indicate the force termination. */
3751                 f_quit = 1;
3752                 /* exit with the expected status */
3753                 signal(signum, SIG_DFL);
3754                 kill(getpid(), signum);
3755         }
3756 }
3757
3758 int
3759 main(int argc, char** argv)
3760 {
3761         int diag;
3762         portid_t port_id;
3763         uint16_t count;
3764         int ret;
3765
3766         signal(SIGINT, signal_handler);
3767         signal(SIGTERM, signal_handler);
3768
3769         testpmd_logtype = rte_log_register("testpmd");
3770         if (testpmd_logtype < 0)
3771                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3772         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3773
3774         diag = rte_eal_init(argc, argv);
3775         if (diag < 0)
3776                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3777                          rte_strerror(rte_errno));
3778
3779         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3780                 rte_exit(EXIT_FAILURE,
3781                          "Secondary process type not supported.\n");
3782
3783         ret = register_eth_event_callback();
3784         if (ret != 0)
3785                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3786
3787 #ifdef RTE_LIB_PDUMP
3788         /* initialize packet capture framework */
3789         rte_pdump_init();
3790 #endif
3791
3792         count = 0;
3793         RTE_ETH_FOREACH_DEV(port_id) {
3794                 ports_ids[count] = port_id;
3795                 count++;
3796         }
3797         nb_ports = (portid_t) count;
3798         if (nb_ports == 0)
3799                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3800
3801         /* allocate port structures, and init them */
3802         init_port();
3803
3804         set_def_fwd_config();
3805         if (nb_lcores == 0)
3806                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3807                          "Check the core mask argument\n");
3808
3809         /* Bitrate/latency stats disabled by default */
3810 #ifdef RTE_LIB_BITRATESTATS
3811         bitrate_enabled = 0;
3812 #endif
3813 #ifdef RTE_LIB_LATENCYSTATS
3814         latencystats_enabled = 0;
3815 #endif
3816
3817         /* on FreeBSD, mlockall() is disabled by default */
3818 #ifdef RTE_EXEC_ENV_FREEBSD
3819         do_mlockall = 0;
3820 #else
3821         do_mlockall = 1;
3822 #endif
3823
3824         argc -= diag;
3825         argv += diag;
3826         if (argc > 1)
3827                 launch_args_parse(argc, argv);
3828
3829         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3830                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3831                         strerror(errno));
3832         }
3833
3834         if (tx_first && interactive)
3835                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3836                                 "interactive mode.\n");
3837
3838         if (tx_first && lsc_interrupt) {
3839                 printf("Warning: lsc_interrupt needs to be off when "
3840                                 " using tx_first. Disabling.\n");
3841                 lsc_interrupt = 0;
3842         }
3843
3844         if (!nb_rxq && !nb_txq)
3845                 printf("Warning: Either rx or tx queues should be non-zero\n");
3846
3847         if (nb_rxq > 1 && nb_rxq > nb_txq)
3848                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3849                        "but nb_txq=%d will prevent to fully test it.\n",
3850                        nb_rxq, nb_txq);
3851
3852         init_config();
3853
3854         if (hot_plug) {
3855                 ret = rte_dev_hotplug_handle_enable();
3856                 if (ret) {
3857                         RTE_LOG(ERR, EAL,
3858                                 "fail to enable hotplug handling.");
3859                         return -1;
3860                 }
3861
3862                 ret = rte_dev_event_monitor_start();
3863                 if (ret) {
3864                         RTE_LOG(ERR, EAL,
3865                                 "fail to start device event monitoring.");
3866                         return -1;
3867                 }
3868
3869                 ret = rte_dev_event_callback_register(NULL,
3870                         dev_event_callback, NULL);
3871                 if (ret) {
3872                         RTE_LOG(ERR, EAL,
3873                                 "fail  to register device event callback\n");
3874                         return -1;
3875                 }
3876         }
3877
3878         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3879                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3880
3881         /* set all ports to promiscuous mode by default */
3882         RTE_ETH_FOREACH_DEV(port_id) {
3883                 ret = rte_eth_promiscuous_enable(port_id);
3884                 if (ret != 0)
3885                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3886                                 port_id, rte_strerror(-ret));
3887         }
3888
3889         /* Init metrics library */
3890         rte_metrics_init(rte_socket_id());
3891
3892 #ifdef RTE_LIB_LATENCYSTATS
3893         if (latencystats_enabled != 0) {
3894                 int ret = rte_latencystats_init(1, NULL);
3895                 if (ret)
3896                         printf("Warning: latencystats init()"
3897                                 " returned error %d\n", ret);
3898                 printf("Latencystats running on lcore %d\n",
3899                         latencystats_lcore_id);
3900         }
3901 #endif
3902
3903         /* Setup bitrate stats */
3904 #ifdef RTE_LIB_BITRATESTATS
3905         if (bitrate_enabled != 0) {
3906                 bitrate_data = rte_stats_bitrate_create();
3907                 if (bitrate_data == NULL)
3908                         rte_exit(EXIT_FAILURE,
3909                                 "Could not allocate bitrate data.\n");
3910                 rte_stats_bitrate_reg(bitrate_data);
3911         }
3912 #endif
3913
3914 #ifdef RTE_LIB_CMDLINE
3915         if (strlen(cmdline_filename) != 0)
3916                 cmdline_read_from_file(cmdline_filename);
3917
3918         if (interactive == 1) {
3919                 if (auto_start) {
3920                         printf("Start automatic packet forwarding\n");
3921                         start_packet_forwarding(0);
3922                 }
3923                 prompt();
3924                 pmd_test_exit();
3925         } else
3926 #endif
3927         {
3928                 char c;
3929                 int rc;
3930
3931                 f_quit = 0;
3932
3933                 printf("No commandline core given, start packet forwarding\n");
3934                 start_packet_forwarding(tx_first);
3935                 if (stats_period != 0) {
3936                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3937                         uint64_t timer_period;
3938
3939                         /* Convert to number of cycles */
3940                         timer_period = stats_period * rte_get_timer_hz();
3941
3942                         while (f_quit == 0) {
3943                                 cur_time = rte_get_timer_cycles();
3944                                 diff_time += cur_time - prev_time;
3945
3946                                 if (diff_time >= timer_period) {
3947                                         print_stats();
3948                                         /* Reset the timer */
3949                                         diff_time = 0;
3950                                 }
3951                                 /* Sleep to avoid unnecessary checks */
3952                                 prev_time = cur_time;
3953                                 sleep(1);
3954                         }
3955                 }
3956
3957                 printf("Press enter to exit\n");
3958                 rc = read(0, &c, 1);
3959                 pmd_test_exit();
3960                 if (rc < 0)
3961                         return 1;
3962         }
3963
3964         ret = rte_eal_cleanup();
3965         if (ret != 0)
3966                 rte_exit(EXIT_FAILURE,
3967                          "EAL cleanup failed: %s\n", strerror(-ret));
3968
3969         return EXIT_SUCCESS;
3970 }