app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 199 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 200                                       * specified on command-line. */
 201 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 202
 203 /*
 204  * In container, it cannot terminate the process which running with 'stats-period'
 205  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 206  */
 207 uint8_t f_quit;
 208
 209 /*
 210  * Configuration of packet segments used by the "txonly" processing engine.
 211  */
 212 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 213 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 214         TXONLY_DEF_PACKET_LEN,
 215 };
 216 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 217
 218 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 219 /**< Split policy for packets to TX. */
 220
 221 uint8_t txonly_multi_flow;
 222 /**< Whether multiple flows are generated in TXONLY mode. */
 223
 224 uint32_t tx_pkt_times_inter;
 225 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 226
 227 uint32_t tx_pkt_times_intra;
 228 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 229
 230 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 231 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 232
 233 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 234 uint8_t dcb_config = 0;
 235
 236 /* Whether the dcb is in testing status */
 237 uint8_t dcb_test = 0;
 238
 239 /*
 240  * Configurable number of RX/TX queues.
 241  */
 242 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 243 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 244 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 245
 246 /*
 247  * Configurable number of RX/TX ring descriptors.
 248  * Defaults are supplied by drivers via ethdev.
 249  */
 250 #define RTE_TEST_RX_DESC_DEFAULT 0
 251 #define RTE_TEST_TX_DESC_DEFAULT 0
 252 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 253 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 254
 255 #define RTE_PMD_PARAM_UNSET -1
 256 /*
 257  * Configurable values of RX and TX ring threshold registers.
 258  */
 259
 260 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 262 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 263
 264 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 265 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 266 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX free threshold.
 270  */
 271 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of RX drop enable.
 275  */
 276 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX free threshold.
 280  */
 281 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of TX RS bit threshold.
 285  */
 286 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of buffered packets before sending.
 290  */
 291 uint16_t noisy_tx_sw_bufsz;
 292
 293 /*
 294  * Configurable value of packet buffer timeout.
 295  */
 296 uint16_t noisy_tx_sw_buf_flush_time;
 297
 298 /*
 299  * Configurable value for size of VNF internal memory area
 300  * used for simulating noisy neighbour behaviour
 301  */
 302 uint64_t noisy_lkup_mem_sz;
 303
 304 /*
 305  * Configurable value of number of random writes done in
 306  * VNF simulation memory area.
 307  */
 308 uint64_t noisy_lkup_num_writes;
 309
 310 /*
 311  * Configurable value of number of random reads done in
 312  * VNF simulation memory area.
 313  */
 314 uint64_t noisy_lkup_num_reads;
 315
 316 /*
 317  * Configurable value of number of random reads/writes done in
 318  * VNF simulation memory area.
 319  */
 320 uint64_t noisy_lkup_num_reads_writes;
 321
 322 /*
 323  * Receive Side Scaling (RSS) configuration.
 324  */
 325 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 326
 327 /*
 328  * Port topology configuration
 329  */
 330 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 331
 332 /*
 333  * Avoids to flush all the RX streams before starts forwarding.
 334  */
 335 uint8_t no_flush_rx = 0; /* flush by default */
 336
 337 /*
 338  * Flow API isolated mode.
 339  */
 340 uint8_t flow_isolate_all;
 341
 342 /*
 343  * Avoids to check link status when starting/stopping a port.
 344  */
 345 uint8_t no_link_check = 0; /* check by default */
 346
 347 /*
 348  * Don't automatically start all ports in interactive mode.
 349  */
 350 uint8_t no_device_start = 0;
 351
 352 /*
 353  * Enable link status change notification
 354  */
 355 uint8_t lsc_interrupt = 1; /* enabled by default */
 356
 357 /*
 358  * Enable device removal notification.
 359  */
 360 uint8_t rmv_interrupt = 1; /* enabled by default */
 361
 362 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 363
 364 /* After attach, port setup is called on event or by iterator */
 365 bool setup_on_probe_event = true;
 366
 367 /* Clear ptypes on port initialization. */
 368 uint8_t clear_ptypes = true;
 369
 370 /* Pretty printing of ethdev events */
 371 static const char * const eth_event_desc[] = {
 372         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 373         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 374         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 375         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 376         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 377         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 378         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 379         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 380         [RTE_ETH_EVENT_NEW] = "device probed",
 381         [RTE_ETH_EVENT_DESTROY] = "device released",
 382         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 383         [RTE_ETH_EVENT_MAX] = NULL,
 384 };
 385
 386 /*
 387  * Display or mask ether events
 388  * Default to all events except VF_MBOX
 389  */
 390 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 391                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 392                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 393                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 394                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 395                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 396                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 397                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 398 /*
 399  * Decide if all memory are locked for performance.
 400  */
 401 int do_mlockall = 0;
 402
 403 /*
 404  * NIC bypass mode configuration options.
 405  */
 406
 407 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 408 /* The NIC bypass watchdog timeout. */
 409 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 410 #endif
 411
 412
 413 #ifdef RTE_LIBRTE_LATENCY_STATS
 414
 415 /*
 416  * Set when latency stats is enabled in the commandline
 417  */
 418 uint8_t latencystats_enabled;
 419
 420 /*
 421  * Lcore ID to serive latency statistics.
 422  */
 423 lcoreid_t latencystats_lcore_id = -1;
 424
 425 #endif
 426
 427 /*
 428  * Ethernet device configuration.
 429  */
 430 struct rte_eth_rxmode rx_mode = {
 431         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 432                 /**< Default maximum frame length. */
 433 };
 434
 435 struct rte_eth_txmode tx_mode = {
 436         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 437 };
 438
 439 struct rte_fdir_conf fdir_conf = {
 440         .mode = RTE_FDIR_MODE_NONE,
 441         .pballoc = RTE_FDIR_PBALLOC_64K,
 442         .status = RTE_FDIR_REPORT_STATUS,
 443         .mask = {
 444                 .vlan_tci_mask = 0xFFEF,
 445                 .ipv4_mask     = {
 446                         .src_ip = 0xFFFFFFFF,
 447                         .dst_ip = 0xFFFFFFFF,
 448                 },
 449                 .ipv6_mask     = {
 450                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 451                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 452                 },
 453                 .src_port_mask = 0xFFFF,
 454                 .dst_port_mask = 0xFFFF,
 455                 .mac_addr_byte_mask = 0xFF,
 456                 .tunnel_type_mask = 1,
 457                 .tunnel_id_mask = 0xFFFFFFFF,
 458         },
 459         .drop_queue = 127,
 460 };
 461
 462 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 463
 464 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 465 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 466
 467 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 468 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 469
 470 uint16_t nb_tx_queue_stats_mappings = 0;
 471 uint16_t nb_rx_queue_stats_mappings = 0;
 472
 473 /*
 474  * Display zero values by default for xstats
 475  */
 476 uint8_t xstats_hide_zero;
 477
 478 /*
 479  * Measure of CPU cycles disabled by default
 480  */
 481 uint8_t record_core_cycles;
 482
 483 unsigned int num_sockets = 0;
 484 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 485
 486 #ifdef RTE_LIBRTE_BITRATE
 487 /* Bitrate statistics */
 488 struct rte_stats_bitrates *bitrate_data;
 489 lcoreid_t bitrate_lcore_id;
 490 uint8_t bitrate_enabled;
 491 #endif
 492
 493 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 494 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 495
 496 /*
 497  * hexadecimal bitmask of RX mq mode can be enabled.
 498  */
 499 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 500
 501 /* Forward function declarations */
 502 static void setup_attached_port(portid_t pi);
 503 static void map_port_queue_stats_mapping_registers(portid_t pi,
 504                                                    struct rte_port *port);
 505 static void check_all_ports_link_status(uint32_t port_mask);
 506 static int eth_event_callback(portid_t port_id,
 507                               enum rte_eth_event_type type,
 508                               void *param, void *ret_param);
 509 static void dev_event_callback(const char *device_name,
 510                                 enum rte_dev_event_type type,
 511                                 void *param);
 512
 513 /*
 514  * Check if all the ports are started.
 515  * If yes, return positive value. If not, return zero.
 516  */
 517 static int all_ports_started(void);
 518
 519 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 520 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 521
 522 /* Holds the registered mbuf dynamic flags names. */
 523 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 524
 525 /*
 526  * Helper function to check if socket is already discovered.
 527  * If yes, return positive value. If not, return zero.
 528  */
 529 int
 530 new_socket_id(unsigned int socket_id)
 531 {
 532         unsigned int i;
 533
 534         for (i = 0; i < num_sockets; i++) {
 535                 if (socket_ids[i] == socket_id)
 536                         return 0;
 537         }
 538         return 1;
 539 }
 540
 541 /*
 542  * Setup default configuration.
 543  */
 544 static void
 545 set_default_fwd_lcores_config(void)
 546 {
 547         unsigned int i;
 548         unsigned int nb_lc;
 549         unsigned int sock_num;
 550
 551         nb_lc = 0;
 552         for (i = 0; i < RTE_MAX_LCORE; i++) {
 553                 if (!rte_lcore_is_enabled(i))
 554                         continue;
 555                 sock_num = rte_lcore_to_socket_id(i);
 556                 if (new_socket_id(sock_num)) {
 557                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 558                                 rte_exit(EXIT_FAILURE,
 559                                          "Total sockets greater than %u\n",
 560                                          RTE_MAX_NUMA_NODES);
 561                         }
 562                         socket_ids[num_sockets++] = sock_num;
 563                 }
 564                 if (i == rte_get_master_lcore())
 565                         continue;
 566                 fwd_lcores_cpuids[nb_lc++] = i;
 567         }
 568         nb_lcores = (lcoreid_t) nb_lc;
 569         nb_cfg_lcores = nb_lcores;
 570         nb_fwd_lcores = 1;
 571 }
 572
 573 static void
 574 set_def_peer_eth_addrs(void)
 575 {
 576         portid_t i;
 577
 578         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 579                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 580                 peer_eth_addrs[i].addr_bytes[5] = i;
 581         }
 582 }
 583
 584 static void
 585 set_default_fwd_ports_config(void)
 586 {
 587         portid_t pt_id;
 588         int i = 0;
 589
 590         RTE_ETH_FOREACH_DEV(pt_id) {
 591                 fwd_ports_ids[i++] = pt_id;
 592
 593                 /* Update sockets info according to the attached device */
 594                 int socket_id = rte_eth_dev_socket_id(pt_id);
 595                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 596                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 597                                 rte_exit(EXIT_FAILURE,
 598                                          "Total sockets greater than %u\n",
 599                                          RTE_MAX_NUMA_NODES);
 600                         }
 601                         socket_ids[num_sockets++] = socket_id;
 602                 }
 603         }
 604
 605         nb_cfg_ports = nb_ports;
 606         nb_fwd_ports = nb_ports;
 607 }
 608
 609 void
 610 set_def_fwd_config(void)
 611 {
 612         set_default_fwd_lcores_config();
 613         set_def_peer_eth_addrs();
 614         set_default_fwd_ports_config();
 615 }
 616
 617 /* extremely pessimistic estimation of memory required to create a mempool */
 618 static int
 619 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 620 {
 621         unsigned int n_pages, mbuf_per_pg, leftover;
 622         uint64_t total_mem, mbuf_mem, obj_sz;
 623
 624         /* there is no good way to predict how much space the mempool will
 625          * occupy because it will allocate chunks on the fly, and some of those
 626          * will come from default DPDK memory while some will come from our
 627          * external memory, so just assume 128MB will be enough for everyone.
 628          */
 629         uint64_t hdr_mem = 128 << 20;
 630
 631         /* account for possible non-contiguousness */
 632         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 633         if (obj_sz > pgsz) {
 634                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 635                 return -1;
 636         }
 637
 638         mbuf_per_pg = pgsz / obj_sz;
 639         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 640         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 641
 642         mbuf_mem = n_pages * pgsz;
 643
 644         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 645
 646         if (total_mem > SIZE_MAX) {
 647                 TESTPMD_LOG(ERR, "Memory size too big\n");
 648                 return -1;
 649         }
 650         *out = (size_t)total_mem;
 651
 652         return 0;
 653 }
 654
 655 static int
 656 pagesz_flags(uint64_t page_sz)
 657 {
 658         /* as per mmap() manpage, all page sizes are log2 of page size
 659          * shifted by MAP_HUGE_SHIFT
 660          */
 661         int log2 = rte_log2_u64(page_sz);
 662
 663         return (log2 << HUGE_SHIFT);
 664 }
 665
 666 static void *
 667 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 668 {
 669         void *addr;
 670         int flags;
 671
 672         /* allocate anonymous hugepages */
 673         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 674         if (huge)
 675                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 676
 677         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 678         if (addr == MAP_FAILED)
 679                 return NULL;
 680
 681         return addr;
 682 }
 683
 684 struct extmem_param {
 685         void *addr;
 686         size_t len;
 687         size_t pgsz;
 688         rte_iova_t *iova_table;
 689         unsigned int iova_table_len;
 690 };
 691
 692 static int
 693 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 694                 bool huge)
 695 {
 696         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 697                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 698         unsigned int cur_page, n_pages, pgsz_idx;
 699         size_t mem_sz, cur_pgsz;
 700         rte_iova_t *iovas = NULL;
 701         void *addr;
 702         int ret;
 703
 704         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 705                 /* skip anything that is too big */
 706                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 707                         continue;
 708
 709                 cur_pgsz = pgsizes[pgsz_idx];
 710
 711                 /* if we were told not to allocate hugepages, override */
 712                 if (!huge)
 713                         cur_pgsz = sysconf(_SC_PAGESIZE);
 714
 715                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 716                 if (ret < 0) {
 717                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 718                         return -1;
 719                 }
 720
 721                 /* allocate our memory */
 722                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 723
 724                 /* if we couldn't allocate memory with a specified page size,
 725                  * that doesn't mean we can't do it with other page sizes, so
 726                  * try another one.
 727                  */
 728                 if (addr == NULL)
 729                         continue;
 730
 731                 /* store IOVA addresses for every page in this memory area */
 732                 n_pages = mem_sz / cur_pgsz;
 733
 734                 iovas = malloc(sizeof(*iovas) * n_pages);
 735
 736                 if (iovas == NULL) {
 737                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 738                         goto fail;
 739                 }
 740                 /* lock memory if it's not huge pages */
 741                 if (!huge)
 742                         mlock(addr, mem_sz);
 743
 744                 /* populate IOVA addresses */
 745                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 746                         rte_iova_t iova;
 747                         size_t offset;
 748                         void *cur;
 749
 750                         offset = cur_pgsz * cur_page;
 751                         cur = RTE_PTR_ADD(addr, offset);
 752
 753                         /* touch the page before getting its IOVA */
 754                         *(volatile char *)cur = 0;
 755
 756                         iova = rte_mem_virt2iova(cur);
 757
 758                         iovas[cur_page] = iova;
 759                 }
 760
 761                 break;
 762         }
 763         /* if we couldn't allocate anything */
 764         if (iovas == NULL)
 765                 return -1;
 766
 767         param->addr = addr;
 768         param->len = mem_sz;
 769         param->pgsz = cur_pgsz;
 770         param->iova_table = iovas;
 771         param->iova_table_len = n_pages;
 772
 773         return 0;
 774 fail:
 775         if (iovas)
 776                 free(iovas);
 777         if (addr)
 778                 munmap(addr, mem_sz);
 779
 780         return -1;
 781 }
 782
 783 static int
 784 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 785 {
 786         struct extmem_param param;
 787         int socket_id, ret;
 788
 789         memset(&param, 0, sizeof(param));
 790
 791         /* check if our heap exists */
 792         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 793         if (socket_id < 0) {
 794                 /* create our heap */
 795                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 796                 if (ret < 0) {
 797                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 798                         return -1;
 799                 }
 800         }
 801
 802         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 803         if (ret < 0) {
 804                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 805                 return -1;
 806         }
 807
 808         /* we now have a valid memory area, so add it to heap */
 809         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 810                         param.addr, param.len, param.iova_table,
 811                         param.iova_table_len, param.pgsz);
 812
 813         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 814
 815         /* not needed any more */
 816         free(param.iova_table);
 817
 818         if (ret < 0) {
 819                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 820                 munmap(param.addr, param.len);
 821                 return -1;
 822         }
 823
 824         /* success */
 825
 826         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 827                         param.len >> 20);
 828
 829         return 0;
 830 }
 831 static void
 832 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 833              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 834 {
 835         uint16_t pid = 0;
 836         int ret;
 837
 838         RTE_ETH_FOREACH_DEV(pid) {
 839                 struct rte_eth_dev *dev =
 840                         &rte_eth_devices[pid];
 841
 842                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 843                                         memhdr->len);
 844                 if (ret) {
 845                         TESTPMD_LOG(DEBUG,
 846                                     "unable to DMA unmap addr 0x%p "
 847                                     "for device %s\n",
 848                                     memhdr->addr, dev->data->name);
 849                 }
 850         }
 851         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 852         if (ret) {
 853                 TESTPMD_LOG(DEBUG,
 854                             "unable to un-register addr 0x%p\n", memhdr->addr);
 855         }
 856 }
 857
 858 static void
 859 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 860            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 861 {
 862         uint16_t pid = 0;
 863         size_t page_size = sysconf(_SC_PAGESIZE);
 864         int ret;
 865
 866         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 867                                   page_size);
 868         if (ret) {
 869                 TESTPMD_LOG(DEBUG,
 870                             "unable to register addr 0x%p\n", memhdr->addr);
 871                 return;
 872         }
 873         RTE_ETH_FOREACH_DEV(pid) {
 874                 struct rte_eth_dev *dev =
 875                         &rte_eth_devices[pid];
 876
 877                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 878                                       memhdr->len);
 879                 if (ret) {
 880                         TESTPMD_LOG(DEBUG,
 881                                     "unable to DMA map addr 0x%p "
 882                                     "for device %s\n",
 883                                     memhdr->addr, dev->data->name);
 884                 }
 885         }
 886 }
 887
 888 static unsigned int
 889 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 890             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 891 {
 892         struct rte_pktmbuf_extmem *xmem;
 893         unsigned int ext_num, zone_num, elt_num;
 894         uint16_t elt_size;
 895
 896         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 897         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 898         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 899
 900         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 901         if (xmem == NULL) {
 902                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 903                                  "external buffer descriptors\n");
 904                 *ext_mem = NULL;
 905                 return 0;
 906         }
 907         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 908                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 909                 const struct rte_memzone *mz;
 910                 char mz_name[RTE_MEMZONE_NAMESIZE];
 911                 int ret;
 912
 913                 ret = snprintf(mz_name, sizeof(mz_name),
 914                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 915                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 916                         errno = ENAMETOOLONG;
 917                         ext_num = 0;
 918                         break;
 919                 }
 920                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 921                                                  socket_id,
 922                                                  RTE_MEMZONE_IOVA_CONTIG |
 923                                                  RTE_MEMZONE_1GB |
 924                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 925                                                  EXTBUF_ZONE_SIZE);
 926                 if (mz == NULL) {
 927                         /*
 928                          * The caller exits on external buffer creation
 929                          * error, so there is no need to free memzones.
 930                          */
 931                         errno = ENOMEM;
 932                         ext_num = 0;
 933                         break;
 934                 }
 935                 xseg->buf_ptr = mz->addr;
 936                 xseg->buf_iova = mz->iova;
 937                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 938                 xseg->elt_size = elt_size;
 939         }
 940         if (ext_num == 0 && xmem != NULL) {
 941                 free(xmem);
 942                 xmem = NULL;
 943         }
 944         *ext_mem = xmem;
 945         return ext_num;
 946 }
 947
 948 /*
 949  * Configuration initialisation done once at init time.
 950  */
 951 static struct rte_mempool *
 952 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 953                  unsigned int socket_id)
 954 {
 955         char pool_name[RTE_MEMPOOL_NAMESIZE];
 956         struct rte_mempool *rte_mp = NULL;
 957         uint32_t mb_size;
 958
 959         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 960         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 961
 962         TESTPMD_LOG(INFO,
 963                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 964                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 965
 966         switch (mp_alloc_type) {
 967         case MP_ALLOC_NATIVE:
 968                 {
 969                         /* wrapper to rte_mempool_create() */
 970                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 971                                         rte_mbuf_best_mempool_ops());
 972                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 973                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 974                         break;
 975                 }
 976         case MP_ALLOC_ANON:
 977                 {
 978                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 979                                 mb_size, (unsigned int) mb_mempool_cache,
 980                                 sizeof(struct rte_pktmbuf_pool_private),
 981                                 socket_id, mempool_flags);
 982                         if (rte_mp == NULL)
 983                                 goto err;
 984
 985                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 986                                 rte_mempool_free(rte_mp);
 987                                 rte_mp = NULL;
 988                                 goto err;
 989                         }
 990                         rte_pktmbuf_pool_init(rte_mp, NULL);
 991                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 992                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 993                         break;
 994                 }
 995         case MP_ALLOC_XMEM:
 996         case MP_ALLOC_XMEM_HUGE:
 997                 {
 998                         int heap_socket;
 999                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1000
1001                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1002                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1003
1004                         heap_socket =
1005                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1006                         if (heap_socket < 0)
1007                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1008
1009                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1010                                         rte_mbuf_best_mempool_ops());
1011                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1012                                         mb_mempool_cache, 0, mbuf_seg_size,
1013                                         heap_socket);
1014                         break;
1015                 }
1016         case MP_ALLOC_XBUF:
1017                 {
1018                         struct rte_pktmbuf_extmem *ext_mem;
1019                         unsigned int ext_num;
1020
1021                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1022                                                socket_id, pool_name, &ext_mem);
1023                         if (ext_num == 0)
1024                                 rte_exit(EXIT_FAILURE,
1025                                          "Can't create pinned data buffers\n");
1026
1027                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1028                                         rte_mbuf_best_mempool_ops());
1029                         rte_mp = rte_pktmbuf_pool_create_extbuf
1030                                         (pool_name, nb_mbuf, mb_mempool_cache,
1031                                          0, mbuf_seg_size, socket_id,
1032                                          ext_mem, ext_num);
1033                         free(ext_mem);
1034                         break;
1035                 }
1036         default:
1037                 {
1038                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1039                 }
1040         }
1041
1042 err:
1043         if (rte_mp == NULL) {
1044                 rte_exit(EXIT_FAILURE,
1045                         "Creation of mbuf pool for socket %u failed: %s\n",
1046                         socket_id, rte_strerror(rte_errno));
1047         } else if (verbose_level > 0) {
1048                 rte_mempool_dump(stdout, rte_mp);
1049         }
1050         return rte_mp;
1051 }
1052
1053 /*
1054  * Check given socket id is valid or not with NUMA mode,
1055  * if valid, return 0, else return -1
1056  */
1057 static int
1058 check_socket_id(const unsigned int socket_id)
1059 {
1060         static int warning_once = 0;
1061
1062         if (new_socket_id(socket_id)) {
1063                 if (!warning_once && numa_support)
1064                         printf("Warning: NUMA should be configured manually by"
1065                                " using --port-numa-config and"
1066                                " --ring-numa-config parameters along with"
1067                                " --numa.\n");
1068                 warning_once = 1;
1069                 return -1;
1070         }
1071         return 0;
1072 }
1073
1074 /*
1075  * Get the allowed maximum number of RX queues.
1076  * *pid return the port id which has minimal value of
1077  * max_rx_queues in all ports.
1078  */
1079 queueid_t
1080 get_allowed_max_nb_rxq(portid_t *pid)
1081 {
1082         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1083         bool max_rxq_valid = false;
1084         portid_t pi;
1085         struct rte_eth_dev_info dev_info;
1086
1087         RTE_ETH_FOREACH_DEV(pi) {
1088                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1089                         continue;
1090
1091                 max_rxq_valid = true;
1092                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1093                         allowed_max_rxq = dev_info.max_rx_queues;
1094                         *pid = pi;
1095                 }
1096         }
1097         return max_rxq_valid ? allowed_max_rxq : 0;
1098 }
1099
1100 /*
1101  * Check input rxq is valid or not.
1102  * If input rxq is not greater than any of maximum number
1103  * of RX queues of all ports, it is valid.
1104  * if valid, return 0, else return -1
1105  */
1106 int
1107 check_nb_rxq(queueid_t rxq)
1108 {
1109         queueid_t allowed_max_rxq;
1110         portid_t pid = 0;
1111
1112         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1113         if (rxq > allowed_max_rxq) {
1114                 printf("Fail: input rxq (%u) can't be greater "
1115                        "than max_rx_queues (%u) of port %u\n",
1116                        rxq,
1117                        allowed_max_rxq,
1118                        pid);
1119                 return -1;
1120         }
1121         return 0;
1122 }
1123
1124 /*
1125  * Get the allowed maximum number of TX queues.
1126  * *pid return the port id which has minimal value of
1127  * max_tx_queues in all ports.
1128  */
1129 queueid_t
1130 get_allowed_max_nb_txq(portid_t *pid)
1131 {
1132         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1133         bool max_txq_valid = false;
1134         portid_t pi;
1135         struct rte_eth_dev_info dev_info;
1136
1137         RTE_ETH_FOREACH_DEV(pi) {
1138                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1139                         continue;
1140
1141                 max_txq_valid = true;
1142                 if (dev_info.max_tx_queues < allowed_max_txq) {
1143                         allowed_max_txq = dev_info.max_tx_queues;
1144                         *pid = pi;
1145                 }
1146         }
1147         return max_txq_valid ? allowed_max_txq : 0;
1148 }
1149
1150 /*
1151  * Check input txq is valid or not.
1152  * If input txq is not greater than any of maximum number
1153  * of TX queues of all ports, it is valid.
1154  * if valid, return 0, else return -1
1155  */
1156 int
1157 check_nb_txq(queueid_t txq)
1158 {
1159         queueid_t allowed_max_txq;
1160         portid_t pid = 0;
1161
1162         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1163         if (txq > allowed_max_txq) {
1164                 printf("Fail: input txq (%u) can't be greater "
1165                        "than max_tx_queues (%u) of port %u\n",
1166                        txq,
1167                        allowed_max_txq,
1168                        pid);
1169                 return -1;
1170         }
1171         return 0;
1172 }
1173
1174 /*
1175  * Get the allowed maximum number of RXDs of every rx queue.
1176  * *pid return the port id which has minimal value of
1177  * max_rxd in all queues of all ports.
1178  */
1179 static uint16_t
1180 get_allowed_max_nb_rxd(portid_t *pid)
1181 {
1182         uint16_t allowed_max_rxd = UINT16_MAX;
1183         portid_t pi;
1184         struct rte_eth_dev_info dev_info;
1185
1186         RTE_ETH_FOREACH_DEV(pi) {
1187                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1188                         continue;
1189
1190                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1191                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1192                         *pid = pi;
1193                 }
1194         }
1195         return allowed_max_rxd;
1196 }
1197
1198 /*
1199  * Get the allowed minimal number of RXDs of every rx queue.
1200  * *pid return the port id which has minimal value of
1201  * min_rxd in all queues of all ports.
1202  */
1203 static uint16_t
1204 get_allowed_min_nb_rxd(portid_t *pid)
1205 {
1206         uint16_t allowed_min_rxd = 0;
1207         portid_t pi;
1208         struct rte_eth_dev_info dev_info;
1209
1210         RTE_ETH_FOREACH_DEV(pi) {
1211                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1212                         continue;
1213
1214                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1215                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1216                         *pid = pi;
1217                 }
1218         }
1219
1220         return allowed_min_rxd;
1221 }
1222
1223 /*
1224  * Check input rxd is valid or not.
1225  * If input rxd is not greater than any of maximum number
1226  * of RXDs of every Rx queues and is not less than any of
1227  * minimal number of RXDs of every Rx queues, it is valid.
1228  * if valid, return 0, else return -1
1229  */
1230 int
1231 check_nb_rxd(queueid_t rxd)
1232 {
1233         uint16_t allowed_max_rxd;
1234         uint16_t allowed_min_rxd;
1235         portid_t pid = 0;
1236
1237         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1238         if (rxd > allowed_max_rxd) {
1239                 printf("Fail: input rxd (%u) can't be greater "
1240                        "than max_rxds (%u) of port %u\n",
1241                        rxd,
1242                        allowed_max_rxd,
1243                        pid);
1244                 return -1;
1245         }
1246
1247         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1248         if (rxd < allowed_min_rxd) {
1249                 printf("Fail: input rxd (%u) can't be less "
1250                        "than min_rxds (%u) of port %u\n",
1251                        rxd,
1252                        allowed_min_rxd,
1253                        pid);
1254                 return -1;
1255         }
1256
1257         return 0;
1258 }
1259
1260 /*
1261  * Get the allowed maximum number of TXDs of every rx queues.
1262  * *pid return the port id which has minimal value of
1263  * max_txd in every tx queue.
1264  */
1265 static uint16_t
1266 get_allowed_max_nb_txd(portid_t *pid)
1267 {
1268         uint16_t allowed_max_txd = UINT16_MAX;
1269         portid_t pi;
1270         struct rte_eth_dev_info dev_info;
1271
1272         RTE_ETH_FOREACH_DEV(pi) {
1273                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1274                         continue;
1275
1276                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1277                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1278                         *pid = pi;
1279                 }
1280         }
1281         return allowed_max_txd;
1282 }
1283
1284 /*
1285  * Get the allowed maximum number of TXDs of every tx queues.
1286  * *pid return the port id which has minimal value of
1287  * min_txd in every tx queue.
1288  */
1289 static uint16_t
1290 get_allowed_min_nb_txd(portid_t *pid)
1291 {
1292         uint16_t allowed_min_txd = 0;
1293         portid_t pi;
1294         struct rte_eth_dev_info dev_info;
1295
1296         RTE_ETH_FOREACH_DEV(pi) {
1297                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1298                         continue;
1299
1300                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1301                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1302                         *pid = pi;
1303                 }
1304         }
1305
1306         return allowed_min_txd;
1307 }
1308
1309 /*
1310  * Check input txd is valid or not.
1311  * If input txd is not greater than any of maximum number
1312  * of TXDs of every Rx queues, it is valid.
1313  * if valid, return 0, else return -1
1314  */
1315 int
1316 check_nb_txd(queueid_t txd)
1317 {
1318         uint16_t allowed_max_txd;
1319         uint16_t allowed_min_txd;
1320         portid_t pid = 0;
1321
1322         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1323         if (txd > allowed_max_txd) {
1324                 printf("Fail: input txd (%u) can't be greater "
1325                        "than max_txds (%u) of port %u\n",
1326                        txd,
1327                        allowed_max_txd,
1328                        pid);
1329                 return -1;
1330         }
1331
1332         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1333         if (txd < allowed_min_txd) {
1334                 printf("Fail: input txd (%u) can't be less "
1335                        "than min_txds (%u) of port %u\n",
1336                        txd,
1337                        allowed_min_txd,
1338                        pid);
1339                 return -1;
1340         }
1341         return 0;
1342 }
1343
1344
1345 /*
1346  * Get the allowed maximum number of hairpin queues.
1347  * *pid return the port id which has minimal value of
1348  * max_hairpin_queues in all ports.
1349  */
1350 queueid_t
1351 get_allowed_max_nb_hairpinq(portid_t *pid)
1352 {
1353         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1354         portid_t pi;
1355         struct rte_eth_hairpin_cap cap;
1356
1357         RTE_ETH_FOREACH_DEV(pi) {
1358                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1359                         *pid = pi;
1360                         return 0;
1361                 }
1362                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1363                         allowed_max_hairpinq = cap.max_nb_queues;
1364                         *pid = pi;
1365                 }
1366         }
1367         return allowed_max_hairpinq;
1368 }
1369
1370 /*
1371  * Check input hairpin is valid or not.
1372  * If input hairpin is not greater than any of maximum number
1373  * of hairpin queues of all ports, it is valid.
1374  * if valid, return 0, else return -1
1375  */
1376 int
1377 check_nb_hairpinq(queueid_t hairpinq)
1378 {
1379         queueid_t allowed_max_hairpinq;
1380         portid_t pid = 0;
1381
1382         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1383         if (hairpinq > allowed_max_hairpinq) {
1384                 printf("Fail: input hairpin (%u) can't be greater "
1385                        "than max_hairpin_queues (%u) of port %u\n",
1386                        hairpinq, allowed_max_hairpinq, pid);
1387                 return -1;
1388         }
1389         return 0;
1390 }
1391
1392 static void
1393 init_config(void)
1394 {
1395         portid_t pid;
1396         struct rte_port *port;
1397         struct rte_mempool *mbp;
1398         unsigned int nb_mbuf_per_pool;
1399         lcoreid_t  lc_id;
1400         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1401         struct rte_gro_param gro_param;
1402         uint32_t gso_types;
1403         uint16_t data_size;
1404         bool warning = 0;
1405         int k;
1406         int ret;
1407
1408         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1409
1410         /* Configuration of logical cores. */
1411         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1412                                 sizeof(struct fwd_lcore *) * nb_lcores,
1413                                 RTE_CACHE_LINE_SIZE);
1414         if (fwd_lcores == NULL) {
1415                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1416                                                         "failed\n", nb_lcores);
1417         }
1418         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1419                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1420                                                sizeof(struct fwd_lcore),
1421                                                RTE_CACHE_LINE_SIZE);
1422                 if (fwd_lcores[lc_id] == NULL) {
1423                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1424                                                                 "failed\n");
1425                 }
1426                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1427         }
1428
1429         RTE_ETH_FOREACH_DEV(pid) {
1430                 port = &ports[pid];
1431                 /* Apply default TxRx configuration for all ports */
1432                 port->dev_conf.txmode = tx_mode;
1433                 port->dev_conf.rxmode = rx_mode;
1434
1435                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1436                 if (ret != 0)
1437                         rte_exit(EXIT_FAILURE,
1438                                  "rte_eth_dev_info_get() failed\n");
1439
1440                 if (!(port->dev_info.tx_offload_capa &
1441                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1442                         port->dev_conf.txmode.offloads &=
1443                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1444                 if (numa_support) {
1445                         if (port_numa[pid] != NUMA_NO_CONFIG)
1446                                 port_per_socket[port_numa[pid]]++;
1447                         else {
1448                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1449
1450                                 /*
1451                                  * if socket_id is invalid,
1452                                  * set to the first available socket.
1453                                  */
1454                                 if (check_socket_id(socket_id) < 0)
1455                                         socket_id = socket_ids[0];
1456                                 port_per_socket[socket_id]++;
1457                         }
1458                 }
1459
1460                 /* Apply Rx offloads configuration */
1461                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1462                         port->rx_conf[k].offloads =
1463                                 port->dev_conf.rxmode.offloads;
1464                 /* Apply Tx offloads configuration */
1465                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1466                         port->tx_conf[k].offloads =
1467                                 port->dev_conf.txmode.offloads;
1468
1469                 /* set flag to initialize port/queue */
1470                 port->need_reconfig = 1;
1471                 port->need_reconfig_queues = 1;
1472                 port->tx_metadata = 0;
1473
1474                 /* Check for maximum number of segments per MTU. Accordingly
1475                  * update the mbuf data size.
1476                  */
1477                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1478                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1479                         data_size = rx_mode.max_rx_pkt_len /
1480                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1481
1482                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1483                                                         mbuf_data_size) {
1484                                 mbuf_data_size = data_size +
1485                                                  RTE_PKTMBUF_HEADROOM;
1486                                 warning = 1;
1487                         }
1488                 }
1489         }
1490
1491         if (warning)
1492                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1493                             mbuf_data_size);
1494
1495         /*
1496          * Create pools of mbuf.
1497          * If NUMA support is disabled, create a single pool of mbuf in
1498          * socket 0 memory by default.
1499          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1500          *
1501          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1502          * nb_txd can be configured at run time.
1503          */
1504         if (param_total_num_mbufs)
1505                 nb_mbuf_per_pool = param_total_num_mbufs;
1506         else {
1507                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1508                         (nb_lcores * mb_mempool_cache) +
1509                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1510                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1511         }
1512
1513         if (numa_support) {
1514                 uint8_t i;
1515
1516                 for (i = 0; i < num_sockets; i++)
1517                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1518                                                        nb_mbuf_per_pool,
1519                                                        socket_ids[i]);
1520         } else {
1521                 if (socket_num == UMA_NO_CONFIG)
1522                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1523                                                        nb_mbuf_per_pool, 0);
1524                 else
1525                         mempools[socket_num] = mbuf_pool_create
1526                                                         (mbuf_data_size,
1527                                                          nb_mbuf_per_pool,
1528                                                          socket_num);
1529         }
1530
1531         init_port_config();
1532
1533         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1534                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1535         /*
1536          * Records which Mbuf pool to use by each logical core, if needed.
1537          */
1538         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1539                 mbp = mbuf_pool_find(
1540                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1541
1542                 if (mbp == NULL)
1543                         mbp = mbuf_pool_find(0);
1544                 fwd_lcores[lc_id]->mbp = mbp;
1545                 /* initialize GSO context */
1546                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1547                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1548                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1549                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1550                         RTE_ETHER_CRC_LEN;
1551                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1552         }
1553
1554         /* Configuration of packet forwarding streams. */
1555         if (init_fwd_streams() < 0)
1556                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1557
1558         fwd_config_setup();
1559
1560         /* create a gro context for each lcore */
1561         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1562         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1563         gro_param.max_item_per_flow = MAX_PKT_BURST;
1564         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1565                 gro_param.socket_id = rte_lcore_to_socket_id(
1566                                 fwd_lcores_cpuids[lc_id]);
1567                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1568                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1569                         rte_exit(EXIT_FAILURE,
1570                                         "rte_gro_ctx_create() failed\n");
1571                 }
1572         }
1573 }
1574
1575
1576 void
1577 reconfig(portid_t new_port_id, unsigned socket_id)
1578 {
1579         struct rte_port *port;
1580         int ret;
1581
1582         /* Reconfiguration of Ethernet ports. */
1583         port = &ports[new_port_id];
1584
1585         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1586         if (ret != 0)
1587                 return;
1588
1589         /* set flag to initialize port/queue */
1590         port->need_reconfig = 1;
1591         port->need_reconfig_queues = 1;
1592         port->socket_id = socket_id;
1593
1594         init_port_config();
1595 }
1596
1597
1598 int
1599 init_fwd_streams(void)
1600 {
1601         portid_t pid;
1602         struct rte_port *port;
1603         streamid_t sm_id, nb_fwd_streams_new;
1604         queueid_t q;
1605
1606         /* set socket id according to numa or not */
1607         RTE_ETH_FOREACH_DEV(pid) {
1608                 port = &ports[pid];
1609                 if (nb_rxq > port->dev_info.max_rx_queues) {
1610                         printf("Fail: nb_rxq(%d) is greater than "
1611                                 "max_rx_queues(%d)\n", nb_rxq,
1612                                 port->dev_info.max_rx_queues);
1613                         return -1;
1614                 }
1615                 if (nb_txq > port->dev_info.max_tx_queues) {
1616                         printf("Fail: nb_txq(%d) is greater than "
1617                                 "max_tx_queues(%d)\n", nb_txq,
1618                                 port->dev_info.max_tx_queues);
1619                         return -1;
1620                 }
1621                 if (numa_support) {
1622                         if (port_numa[pid] != NUMA_NO_CONFIG)
1623                                 port->socket_id = port_numa[pid];
1624                         else {
1625                                 port->socket_id = rte_eth_dev_socket_id(pid);
1626
1627                                 /*
1628                                  * if socket_id is invalid,
1629                                  * set to the first available socket.
1630                                  */
1631                                 if (check_socket_id(port->socket_id) < 0)
1632                                         port->socket_id = socket_ids[0];
1633                         }
1634                 }
1635                 else {
1636                         if (socket_num == UMA_NO_CONFIG)
1637                                 port->socket_id = 0;
1638                         else
1639                                 port->socket_id = socket_num;
1640                 }
1641         }
1642
1643         q = RTE_MAX(nb_rxq, nb_txq);
1644         if (q == 0) {
1645                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1646                 return -1;
1647         }
1648         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1649         if (nb_fwd_streams_new == nb_fwd_streams)
1650                 return 0;
1651         /* clear the old */
1652         if (fwd_streams != NULL) {
1653                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1654                         if (fwd_streams[sm_id] == NULL)
1655                                 continue;
1656                         rte_free(fwd_streams[sm_id]);
1657                         fwd_streams[sm_id] = NULL;
1658                 }
1659                 rte_free(fwd_streams);
1660                 fwd_streams = NULL;
1661         }
1662
1663         /* init new */
1664         nb_fwd_streams = nb_fwd_streams_new;
1665         if (nb_fwd_streams) {
1666                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1667                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1668                         RTE_CACHE_LINE_SIZE);
1669                 if (fwd_streams == NULL)
1670                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1671                                  " (struct fwd_stream *)) failed\n",
1672                                  nb_fwd_streams);
1673
1674                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1675                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1676                                 " struct fwd_stream", sizeof(struct fwd_stream),
1677                                 RTE_CACHE_LINE_SIZE);
1678                         if (fwd_streams[sm_id] == NULL)
1679                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1680                                          "(struct fwd_stream) failed\n");
1681                 }
1682         }
1683
1684         return 0;
1685 }
1686
1687 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1688 static void
1689 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1690 {
1691         uint64_t total_burst, sburst;
1692         uint64_t nb_burst;
1693         uint64_t burst_stats[4];
1694         uint16_t pktnb_stats[4];
1695         uint16_t nb_pkt;
1696         int burst_percent[4], sburstp;
1697         int i;
1698
1699         /*
1700          * First compute the total number of packet bursts and the
1701          * two highest numbers of bursts of the same number of packets.
1702          */
1703         memset(&burst_stats, 0x0, sizeof(burst_stats));
1704         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1705
1706         /* Show stats for 0 burst size always */
1707         total_burst = pbs->pkt_burst_spread[0];
1708         burst_stats[0] = pbs->pkt_burst_spread[0];
1709         pktnb_stats[0] = 0;
1710
1711         /* Find the next 2 burst sizes with highest occurrences. */
1712         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1713                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1714
1715                 if (nb_burst == 0)
1716                         continue;
1717
1718                 total_burst += nb_burst;
1719
1720                 if (nb_burst > burst_stats[1]) {
1721                         burst_stats[2] = burst_stats[1];
1722                         pktnb_stats[2] = pktnb_stats[1];
1723                         burst_stats[1] = nb_burst;
1724                         pktnb_stats[1] = nb_pkt;
1725                 } else if (nb_burst > burst_stats[2]) {
1726                         burst_stats[2] = nb_burst;
1727                         pktnb_stats[2] = nb_pkt;
1728                 }
1729         }
1730         if (total_burst == 0)
1731                 return;
1732
1733         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1734         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1735                 if (i == 3) {
1736                         printf("%d%% of other]\n", 100 - sburstp);
1737                         return;
1738                 }
1739
1740                 sburst += burst_stats[i];
1741                 if (sburst == total_burst) {
1742                         printf("%d%% of %d pkts]\n",
1743                                 100 - sburstp, (int) pktnb_stats[i]);
1744                         return;
1745                 }
1746
1747                 burst_percent[i] =
1748                         (double)burst_stats[i] / total_burst * 100;
1749                 printf("%d%% of %d pkts + ",
1750                         burst_percent[i], (int) pktnb_stats[i]);
1751                 sburstp += burst_percent[i];
1752         }
1753 }
1754 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1755
1756 static void
1757 fwd_stream_stats_display(streamid_t stream_id)
1758 {
1759         struct fwd_stream *fs;
1760         static const char *fwd_top_stats_border = "-------";
1761
1762         fs = fwd_streams[stream_id];
1763         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1764             (fs->fwd_dropped == 0))
1765                 return;
1766         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1767                "TX Port=%2d/Queue=%2d %s\n",
1768                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1769                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1770         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1771                " TX-dropped: %-14"PRIu64,
1772                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1773
1774         /* if checksum mode */
1775         if (cur_fwd_eng == &csum_fwd_engine) {
1776                 printf("  RX- bad IP checksum: %-14"PRIu64
1777                        "  Rx- bad L4 checksum: %-14"PRIu64
1778                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1779                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1780                         fs->rx_bad_outer_l4_csum);
1781         } else {
1782                 printf("\n");
1783         }
1784
1785 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1786         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1787         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1788 #endif
1789 }
1790
1791 void
1792 fwd_stats_display(void)
1793 {
1794         static const char *fwd_stats_border = "----------------------";
1795         static const char *acc_stats_border = "+++++++++++++++";
1796         struct {
1797                 struct fwd_stream *rx_stream;
1798                 struct fwd_stream *tx_stream;
1799                 uint64_t tx_dropped;
1800                 uint64_t rx_bad_ip_csum;
1801                 uint64_t rx_bad_l4_csum;
1802                 uint64_t rx_bad_outer_l4_csum;
1803         } ports_stats[RTE_MAX_ETHPORTS];
1804         uint64_t total_rx_dropped = 0;
1805         uint64_t total_tx_dropped = 0;
1806         uint64_t total_rx_nombuf = 0;
1807         struct rte_eth_stats stats;
1808         uint64_t fwd_cycles = 0;
1809         uint64_t total_recv = 0;
1810         uint64_t total_xmit = 0;
1811         struct rte_port *port;
1812         streamid_t sm_id;
1813         portid_t pt_id;
1814         int i;
1815
1816         memset(ports_stats, 0, sizeof(ports_stats));
1817
1818         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1819                 struct fwd_stream *fs = fwd_streams[sm_id];
1820
1821                 if (cur_fwd_config.nb_fwd_streams >
1822                     cur_fwd_config.nb_fwd_ports) {
1823                         fwd_stream_stats_display(sm_id);
1824                 } else {
1825                         ports_stats[fs->tx_port].tx_stream = fs;
1826                         ports_stats[fs->rx_port].rx_stream = fs;
1827                 }
1828
1829                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1830
1831                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1832                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1833                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1834                                 fs->rx_bad_outer_l4_csum;
1835
1836                 if (record_core_cycles)
1837                         fwd_cycles += fs->core_cycles;
1838         }
1839         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1840                 uint8_t j;
1841
1842                 pt_id = fwd_ports_ids[i];
1843                 port = &ports[pt_id];
1844
1845                 rte_eth_stats_get(pt_id, &stats);
1846                 stats.ipackets -= port->stats.ipackets;
1847                 stats.opackets -= port->stats.opackets;
1848                 stats.ibytes -= port->stats.ibytes;
1849                 stats.obytes -= port->stats.obytes;
1850                 stats.imissed -= port->stats.imissed;
1851                 stats.oerrors -= port->stats.oerrors;
1852                 stats.rx_nombuf -= port->stats.rx_nombuf;
1853
1854                 total_recv += stats.ipackets;
1855                 total_xmit += stats.opackets;
1856                 total_rx_dropped += stats.imissed;
1857                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1858                 total_tx_dropped += stats.oerrors;
1859                 total_rx_nombuf  += stats.rx_nombuf;
1860
1861                 printf("\n  %s Forward statistics for port %-2d %s\n",
1862                        fwd_stats_border, pt_id, fwd_stats_border);
1863
1864                 if (!port->rx_queue_stats_mapping_enabled &&
1865                     !port->tx_queue_stats_mapping_enabled) {
1866                         printf("  RX-packets: %-14"PRIu64
1867                                " RX-dropped: %-14"PRIu64
1868                                "RX-total: %-"PRIu64"\n",
1869                                stats.ipackets, stats.imissed,
1870                                stats.ipackets + stats.imissed);
1871
1872                         if (cur_fwd_eng == &csum_fwd_engine)
1873                                 printf("  Bad-ipcsum: %-14"PRIu64
1874                                        " Bad-l4csum: %-14"PRIu64
1875                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1876                                        ports_stats[pt_id].rx_bad_ip_csum,
1877                                        ports_stats[pt_id].rx_bad_l4_csum,
1878                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1879                         if (stats.ierrors + stats.rx_nombuf > 0) {
1880                                 printf("  RX-error: %-"PRIu64"\n",
1881                                        stats.ierrors);
1882                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1883                                        stats.rx_nombuf);
1884                         }
1885
1886                         printf("  TX-packets: %-14"PRIu64
1887                                " TX-dropped: %-14"PRIu64
1888                                "TX-total: %-"PRIu64"\n",
1889                                stats.opackets, ports_stats[pt_id].tx_dropped,
1890                                stats.opackets + ports_stats[pt_id].tx_dropped);
1891                 } else {
1892                         printf("  RX-packets:             %14"PRIu64
1893                                "    RX-dropped:%14"PRIu64
1894                                "    RX-total:%14"PRIu64"\n",
1895                                stats.ipackets, stats.imissed,
1896                                stats.ipackets + stats.imissed);
1897
1898                         if (cur_fwd_eng == &csum_fwd_engine)
1899                                 printf("  Bad-ipcsum:%14"PRIu64
1900                                        "    Bad-l4csum:%14"PRIu64
1901                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1902                                        ports_stats[pt_id].rx_bad_ip_csum,
1903                                        ports_stats[pt_id].rx_bad_l4_csum,
1904                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1905                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1906                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1907                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1908                                        stats.rx_nombuf);
1909                         }
1910
1911                         printf("  TX-packets:             %14"PRIu64
1912                                "    TX-dropped:%14"PRIu64
1913                                "    TX-total:%14"PRIu64"\n",
1914                                stats.opackets, ports_stats[pt_id].tx_dropped,
1915                                stats.opackets + ports_stats[pt_id].tx_dropped);
1916                 }
1917
1918 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1919                 if (ports_stats[pt_id].rx_stream)
1920                         pkt_burst_stats_display("RX",
1921                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1922                 if (ports_stats[pt_id].tx_stream)
1923                         pkt_burst_stats_display("TX",
1924                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1925 #endif
1926
1927                 if (port->rx_queue_stats_mapping_enabled) {
1928                         printf("\n");
1929                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1930                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1931                                        "     RX-errors:%14"PRIu64
1932                                        "    RX-bytes:%14"PRIu64"\n",
1933                                        j, stats.q_ipackets[j],
1934                                        stats.q_errors[j], stats.q_ibytes[j]);
1935                         }
1936                         printf("\n");
1937                 }
1938                 if (port->tx_queue_stats_mapping_enabled) {
1939                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1940                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1941                                        "                                 TX-bytes:%14"
1942                                        PRIu64"\n",
1943                                        j, stats.q_opackets[j],
1944                                        stats.q_obytes[j]);
1945                         }
1946                 }
1947
1948                 printf("  %s--------------------------------%s\n",
1949                        fwd_stats_border, fwd_stats_border);
1950         }
1951
1952         printf("\n  %s Accumulated forward statistics for all ports"
1953                "%s\n",
1954                acc_stats_border, acc_stats_border);
1955         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1956                "%-"PRIu64"\n"
1957                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1958                "%-"PRIu64"\n",
1959                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1960                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1961         if (total_rx_nombuf > 0)
1962                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1963         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1964                "%s\n",
1965                acc_stats_border, acc_stats_border);
1966         if (record_core_cycles) {
1967 #define CYC_PER_MHZ 1E6
1968                 if (total_recv > 0 || total_xmit > 0) {
1969                         uint64_t total_pkts = 0;
1970                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1971                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1972                                 total_pkts = total_xmit;
1973                         else
1974                                 total_pkts = total_recv;
1975
1976                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1977                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1978                                " MHz Clock\n",
1979                                (double) fwd_cycles / total_pkts,
1980                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1981                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1982                 }
1983         }
1984 }
1985
1986 void
1987 fwd_stats_reset(void)
1988 {
1989         streamid_t sm_id;
1990         portid_t pt_id;
1991         int i;
1992
1993         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1994                 pt_id = fwd_ports_ids[i];
1995                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1996         }
1997         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1998                 struct fwd_stream *fs = fwd_streams[sm_id];
1999
2000                 fs->rx_packets = 0;
2001                 fs->tx_packets = 0;
2002                 fs->fwd_dropped = 0;
2003                 fs->rx_bad_ip_csum = 0;
2004                 fs->rx_bad_l4_csum = 0;
2005                 fs->rx_bad_outer_l4_csum = 0;
2006
2007 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
2008                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2009                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2010 #endif
2011                 fs->core_cycles = 0;
2012         }
2013 }
2014
2015 static void
2016 flush_fwd_rx_queues(void)
2017 {
2018         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2019         portid_t  rxp;
2020         portid_t port_id;
2021         queueid_t rxq;
2022         uint16_t  nb_rx;
2023         uint16_t  i;
2024         uint8_t   j;
2025         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2026         uint64_t timer_period;
2027
2028         /* convert to number of cycles */
2029         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2030
2031         for (j = 0; j < 2; j++) {
2032                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2033                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2034                                 port_id = fwd_ports_ids[rxp];
2035                                 /**
2036                                 * testpmd can stuck in the below do while loop
2037                                 * if rte_eth_rx_burst() always returns nonzero
2038                                 * packets. So timer is added to exit this loop
2039                                 * after 1sec timer expiry.
2040                                 */
2041                                 prev_tsc = rte_rdtsc();
2042                                 do {
2043                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2044                                                 pkts_burst, MAX_PKT_BURST);
2045                                         for (i = 0; i < nb_rx; i++)
2046                                                 rte_pktmbuf_free(pkts_burst[i]);
2047
2048                                         cur_tsc = rte_rdtsc();
2049                                         diff_tsc = cur_tsc - prev_tsc;
2050                                         timer_tsc += diff_tsc;
2051                                 } while ((nb_rx > 0) &&
2052                                         (timer_tsc < timer_period));
2053                                 timer_tsc = 0;
2054                         }
2055                 }
2056                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2057         }
2058 }
2059
2060 static void
2061 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2062 {
2063         struct fwd_stream **fsm;
2064         streamid_t nb_fs;
2065         streamid_t sm_id;
2066 #ifdef RTE_LIBRTE_BITRATE
2067         uint64_t tics_per_1sec;
2068         uint64_t tics_datum;
2069         uint64_t tics_current;
2070         uint16_t i, cnt_ports;
2071
2072         cnt_ports = nb_ports;
2073         tics_datum = rte_rdtsc();
2074         tics_per_1sec = rte_get_timer_hz();
2075 #endif
2076         fsm = &fwd_streams[fc->stream_idx];
2077         nb_fs = fc->stream_nb;
2078         do {
2079                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2080                         (*pkt_fwd)(fsm[sm_id]);
2081 #ifdef RTE_LIBRTE_BITRATE
2082                 if (bitrate_enabled != 0 &&
2083                                 bitrate_lcore_id == rte_lcore_id()) {
2084                         tics_current = rte_rdtsc();
2085                         if (tics_current - tics_datum >= tics_per_1sec) {
2086                                 /* Periodic bitrate calculation */
2087                                 for (i = 0; i < cnt_ports; i++)
2088                                         rte_stats_bitrate_calc(bitrate_data,
2089                                                 ports_ids[i]);
2090                                 tics_datum = tics_current;
2091                         }
2092                 }
2093 #endif
2094 #ifdef RTE_LIBRTE_LATENCY_STATS
2095                 if (latencystats_enabled != 0 &&
2096                                 latencystats_lcore_id == rte_lcore_id())
2097                         rte_latencystats_update();
2098 #endif
2099
2100         } while (! fc->stopped);
2101 }
2102
2103 static int
2104 start_pkt_forward_on_core(void *fwd_arg)
2105 {
2106         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2107                              cur_fwd_config.fwd_eng->packet_fwd);
2108         return 0;
2109 }
2110
2111 /*
2112  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2113  * Used to start communication flows in network loopback test configurations.
2114  */
2115 static int
2116 run_one_txonly_burst_on_core(void *fwd_arg)
2117 {
2118         struct fwd_lcore *fwd_lc;
2119         struct fwd_lcore tmp_lcore;
2120
2121         fwd_lc = (struct fwd_lcore *) fwd_arg;
2122         tmp_lcore = *fwd_lc;
2123         tmp_lcore.stopped = 1;
2124         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2125         return 0;
2126 }
2127
2128 /*
2129  * Launch packet forwarding:
2130  *     - Setup per-port forwarding context.
2131  *     - launch logical cores with their forwarding configuration.
2132  */
2133 static void
2134 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2135 {
2136         port_fwd_begin_t port_fwd_begin;
2137         unsigned int i;
2138         unsigned int lc_id;
2139         int diag;
2140
2141         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2142         if (port_fwd_begin != NULL) {
2143                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2144                         (*port_fwd_begin)(fwd_ports_ids[i]);
2145         }
2146         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2147                 lc_id = fwd_lcores_cpuids[i];
2148                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2149                         fwd_lcores[i]->stopped = 0;
2150                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2151                                                      fwd_lcores[i], lc_id);
2152                         if (diag != 0)
2153                                 printf("launch lcore %u failed - diag=%d\n",
2154                                        lc_id, diag);
2155                 }
2156         }
2157 }
2158
2159 /*
2160  * Launch packet forwarding configuration.
2161  */
2162 void
2163 start_packet_forwarding(int with_tx_first)
2164 {
2165         port_fwd_begin_t port_fwd_begin;
2166         port_fwd_end_t  port_fwd_end;
2167         struct rte_port *port;
2168         unsigned int i;
2169         portid_t   pt_id;
2170
2171         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2172                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2173
2174         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2175                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2176
2177         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2178                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2179                 (!nb_rxq || !nb_txq))
2180                 rte_exit(EXIT_FAILURE,
2181                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2182                         cur_fwd_eng->fwd_mode_name);
2183
2184         if (all_ports_started() == 0) {
2185                 printf("Not all ports were started\n");
2186                 return;
2187         }
2188         if (test_done == 0) {
2189                 printf("Packet forwarding already started\n");
2190                 return;
2191         }
2192
2193
2194         if(dcb_test) {
2195                 for (i = 0; i < nb_fwd_ports; i++) {
2196                         pt_id = fwd_ports_ids[i];
2197                         port = &ports[pt_id];
2198                         if (!port->dcb_flag) {
2199                                 printf("In DCB mode, all forwarding ports must "
2200                                        "be configured in this mode.\n");
2201                                 return;
2202                         }
2203                 }
2204                 if (nb_fwd_lcores == 1) {
2205                         printf("In DCB mode,the nb forwarding cores "
2206                                "should be larger than 1.\n");
2207                         return;
2208                 }
2209         }
2210         test_done = 0;
2211
2212         fwd_config_setup();
2213
2214         if(!no_flush_rx)
2215                 flush_fwd_rx_queues();
2216
2217         pkt_fwd_config_display(&cur_fwd_config);
2218         rxtx_config_display();
2219
2220         fwd_stats_reset();
2221         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2222                 pt_id = fwd_ports_ids[i];
2223                 port = &ports[pt_id];
2224                 map_port_queue_stats_mapping_registers(pt_id, port);
2225         }
2226         if (with_tx_first) {
2227                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2228                 if (port_fwd_begin != NULL) {
2229                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2230                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2231                 }
2232                 while (with_tx_first--) {
2233                         launch_packet_forwarding(
2234                                         run_one_txonly_burst_on_core);
2235                         rte_eal_mp_wait_lcore();
2236                 }
2237                 port_fwd_end = tx_only_engine.port_fwd_end;
2238                 if (port_fwd_end != NULL) {
2239                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2240                                 (*port_fwd_end)(fwd_ports_ids[i]);
2241                 }
2242         }
2243         launch_packet_forwarding(start_pkt_forward_on_core);
2244 }
2245
2246 void
2247 stop_packet_forwarding(void)
2248 {
2249         port_fwd_end_t port_fwd_end;
2250         lcoreid_t lc_id;
2251         portid_t pt_id;
2252         int i;
2253
2254         if (test_done) {
2255                 printf("Packet forwarding not started\n");
2256                 return;
2257         }
2258         printf("Telling cores to stop...");
2259         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2260                 fwd_lcores[lc_id]->stopped = 1;
2261         printf("\nWaiting for lcores to finish...\n");
2262         rte_eal_mp_wait_lcore();
2263         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2264         if (port_fwd_end != NULL) {
2265                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2266                         pt_id = fwd_ports_ids[i];
2267                         (*port_fwd_end)(pt_id);
2268                 }
2269         }
2270
2271         fwd_stats_display();
2272
2273         printf("\nDone.\n");
2274         test_done = 1;
2275 }
2276
2277 void
2278 dev_set_link_up(portid_t pid)
2279 {
2280         if (rte_eth_dev_set_link_up(pid) < 0)
2281                 printf("\nSet link up fail.\n");
2282 }
2283
2284 void
2285 dev_set_link_down(portid_t pid)
2286 {
2287         if (rte_eth_dev_set_link_down(pid) < 0)
2288                 printf("\nSet link down fail.\n");
2289 }
2290
2291 static int
2292 all_ports_started(void)
2293 {
2294         portid_t pi;
2295         struct rte_port *port;
2296
2297         RTE_ETH_FOREACH_DEV(pi) {
2298                 port = &ports[pi];
2299                 /* Check if there is a port which is not started */
2300                 if ((port->port_status != RTE_PORT_STARTED) &&
2301                         (port->slave_flag == 0))
2302                         return 0;
2303         }
2304
2305         /* No port is not started */
2306         return 1;
2307 }
2308
2309 int
2310 port_is_stopped(portid_t port_id)
2311 {
2312         struct rte_port *port = &ports[port_id];
2313
2314         if ((port->port_status != RTE_PORT_STOPPED) &&
2315             (port->slave_flag == 0))
2316                 return 0;
2317         return 1;
2318 }
2319
2320 int
2321 all_ports_stopped(void)
2322 {
2323         portid_t pi;
2324
2325         RTE_ETH_FOREACH_DEV(pi) {
2326                 if (!port_is_stopped(pi))
2327                         return 0;
2328         }
2329
2330         return 1;
2331 }
2332
2333 int
2334 port_is_started(portid_t port_id)
2335 {
2336         if (port_id_is_invalid(port_id, ENABLED_WARN))
2337                 return 0;
2338
2339         if (ports[port_id].port_status != RTE_PORT_STARTED)
2340                 return 0;
2341
2342         return 1;
2343 }
2344
2345 /* Configure the Rx and Tx hairpin queues for the selected port. */
2346 static int
2347 setup_hairpin_queues(portid_t pi)
2348 {
2349         queueid_t qi;
2350         struct rte_eth_hairpin_conf hairpin_conf = {
2351                 .peer_count = 1,
2352         };
2353         int i;
2354         int diag;
2355         struct rte_port *port = &ports[pi];
2356
2357         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2358                 hairpin_conf.peers[0].port = pi;
2359                 hairpin_conf.peers[0].queue = i + nb_rxq;
2360                 diag = rte_eth_tx_hairpin_queue_setup
2361                         (pi, qi, nb_txd, &hairpin_conf);
2362                 i++;
2363                 if (diag == 0)
2364                         continue;
2365
2366                 /* Fail to setup rx queue, return */
2367                 if (rte_atomic16_cmpset(&(port->port_status),
2368                                         RTE_PORT_HANDLING,
2369                                         RTE_PORT_STOPPED) == 0)
2370                         printf("Port %d can not be set back "
2371                                         "to stopped\n", pi);
2372                 printf("Fail to configure port %d hairpin "
2373                                 "queues\n", pi);
2374                 /* try to reconfigure queues next time */
2375                 port->need_reconfig_queues = 1;
2376                 return -1;
2377         }
2378         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2379                 hairpin_conf.peers[0].port = pi;
2380                 hairpin_conf.peers[0].queue = i + nb_txq;
2381                 diag = rte_eth_rx_hairpin_queue_setup
2382                         (pi, qi, nb_rxd, &hairpin_conf);
2383                 i++;
2384                 if (diag == 0)
2385                         continue;
2386
2387                 /* Fail to setup rx queue, return */
2388                 if (rte_atomic16_cmpset(&(port->port_status),
2389                                         RTE_PORT_HANDLING,
2390                                         RTE_PORT_STOPPED) == 0)
2391                         printf("Port %d can not be set back "
2392                                         "to stopped\n", pi);
2393                 printf("Fail to configure port %d hairpin "
2394                                 "queues\n", pi);
2395                 /* try to reconfigure queues next time */
2396                 port->need_reconfig_queues = 1;
2397                 return -1;
2398         }
2399         return 0;
2400 }
2401
2402 int
2403 start_port(portid_t pid)
2404 {
2405         int diag, need_check_link_status = -1;
2406         portid_t pi;
2407         queueid_t qi;
2408         struct rte_port *port;
2409         struct rte_ether_addr mac_addr;
2410         struct rte_eth_hairpin_cap cap;
2411
2412         if (port_id_is_invalid(pid, ENABLED_WARN))
2413                 return 0;
2414
2415         if(dcb_config)
2416                 dcb_test = 1;
2417         RTE_ETH_FOREACH_DEV(pi) {
2418                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2419                         continue;
2420
2421                 need_check_link_status = 0;
2422                 port = &ports[pi];
2423                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2424                                                  RTE_PORT_HANDLING) == 0) {
2425                         printf("Port %d is now not stopped\n", pi);
2426                         continue;
2427                 }
2428
2429                 if (port->need_reconfig > 0) {
2430                         port->need_reconfig = 0;
2431
2432                         if (flow_isolate_all) {
2433                                 int ret = port_flow_isolate(pi, 1);
2434                                 if (ret) {
2435                                         printf("Failed to apply isolated"
2436                                                " mode on port %d\n", pi);
2437                                         return -1;
2438                                 }
2439                         }
2440                         configure_rxtx_dump_callbacks(0);
2441                         printf("Configuring Port %d (socket %u)\n", pi,
2442                                         port->socket_id);
2443                         if (nb_hairpinq > 0 &&
2444                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2445                                 printf("Port %d doesn't support hairpin "
2446                                        "queues\n", pi);
2447                                 return -1;
2448                         }
2449                         /* configure port */
2450                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2451                                                      nb_txq + nb_hairpinq,
2452                                                      &(port->dev_conf));
2453                         if (diag != 0) {
2454                                 if (rte_atomic16_cmpset(&(port->port_status),
2455                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2456                                         printf("Port %d can not be set back "
2457                                                         "to stopped\n", pi);
2458                                 printf("Fail to configure port %d\n", pi);
2459                                 /* try to reconfigure port next time */
2460                                 port->need_reconfig = 1;
2461                                 return -1;
2462                         }
2463                 }
2464                 if (port->need_reconfig_queues > 0) {
2465                         port->need_reconfig_queues = 0;
2466                         /* setup tx queues */
2467                         for (qi = 0; qi < nb_txq; qi++) {
2468                                 if ((numa_support) &&
2469                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2470                                         diag = rte_eth_tx_queue_setup(pi, qi,
2471                                                 port->nb_tx_desc[qi],
2472                                                 txring_numa[pi],
2473                                                 &(port->tx_conf[qi]));
2474                                 else
2475                                         diag = rte_eth_tx_queue_setup(pi, qi,
2476                                                 port->nb_tx_desc[qi],
2477                                                 port->socket_id,
2478                                                 &(port->tx_conf[qi]));
2479
2480                                 if (diag == 0)
2481                                         continue;
2482
2483                                 /* Fail to setup tx queue, return */
2484                                 if (rte_atomic16_cmpset(&(port->port_status),
2485                                                         RTE_PORT_HANDLING,
2486                                                         RTE_PORT_STOPPED) == 0)
2487                                         printf("Port %d can not be set back "
2488                                                         "to stopped\n", pi);
2489                                 printf("Fail to configure port %d tx queues\n",
2490                                        pi);
2491                                 /* try to reconfigure queues next time */
2492                                 port->need_reconfig_queues = 1;
2493                                 return -1;
2494                         }
2495                         for (qi = 0; qi < nb_rxq; qi++) {
2496                                 /* setup rx queues */
2497                                 if ((numa_support) &&
2498                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2499                                         struct rte_mempool * mp =
2500                                                 mbuf_pool_find(rxring_numa[pi]);
2501                                         if (mp == NULL) {
2502                                                 printf("Failed to setup RX queue:"
2503                                                         "No mempool allocation"
2504                                                         " on the socket %d\n",
2505                                                         rxring_numa[pi]);
2506                                                 return -1;
2507                                         }
2508
2509                                         diag = rte_eth_rx_queue_setup(pi, qi,
2510                                              port->nb_rx_desc[qi],
2511                                              rxring_numa[pi],
2512                                              &(port->rx_conf[qi]),
2513                                              mp);
2514                                 } else {
2515                                         struct rte_mempool *mp =
2516                                                 mbuf_pool_find(port->socket_id);
2517                                         if (mp == NULL) {
2518                                                 printf("Failed to setup RX queue:"
2519                                                         "No mempool allocation"
2520                                                         " on the socket %d\n",
2521                                                         port->socket_id);
2522                                                 return -1;
2523                                         }
2524                                         diag = rte_eth_rx_queue_setup(pi, qi,
2525                                              port->nb_rx_desc[qi],
2526                                              port->socket_id,
2527                                              &(port->rx_conf[qi]),
2528                                              mp);
2529                                 }
2530                                 if (diag == 0)
2531                                         continue;
2532
2533                                 /* Fail to setup rx queue, return */
2534                                 if (rte_atomic16_cmpset(&(port->port_status),
2535                                                         RTE_PORT_HANDLING,
2536                                                         RTE_PORT_STOPPED) == 0)
2537                                         printf("Port %d can not be set back "
2538                                                         "to stopped\n", pi);
2539                                 printf("Fail to configure port %d rx queues\n",
2540                                        pi);
2541                                 /* try to reconfigure queues next time */
2542                                 port->need_reconfig_queues = 1;
2543                                 return -1;
2544                         }
2545                         /* setup hairpin queues */
2546                         if (setup_hairpin_queues(pi) != 0)
2547                                 return -1;
2548                 }
2549                 configure_rxtx_dump_callbacks(verbose_level);
2550                 if (clear_ptypes) {
2551                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2552                                         NULL, 0);
2553                         if (diag < 0)
2554                                 printf(
2555                                 "Port %d: Failed to disable Ptype parsing\n",
2556                                 pi);
2557                 }
2558
2559                 /* start port */
2560                 if (rte_eth_dev_start(pi) < 0) {
2561                         printf("Fail to start port %d\n", pi);
2562
2563                         /* Fail to setup rx queue, return */
2564                         if (rte_atomic16_cmpset(&(port->port_status),
2565                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2566                                 printf("Port %d can not be set back to "
2567                                                         "stopped\n", pi);
2568                         continue;
2569                 }
2570
2571                 if (rte_atomic16_cmpset(&(port->port_status),
2572                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2573                         printf("Port %d can not be set into started\n", pi);
2574
2575                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2576                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2577                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2578                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2579                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2580
2581                 /* at least one port started, need checking link status */
2582                 need_check_link_status = 1;
2583         }
2584
2585         if (need_check_link_status == 1 && !no_link_check)
2586                 check_all_ports_link_status(RTE_PORT_ALL);
2587         else if (need_check_link_status == 0)
2588                 printf("Please stop the ports first\n");
2589
2590         printf("Done\n");
2591         return 0;
2592 }
2593
2594 void
2595 stop_port(portid_t pid)
2596 {
2597         portid_t pi;
2598         struct rte_port *port;
2599         int need_check_link_status = 0;
2600
2601         if (dcb_test) {
2602                 dcb_test = 0;
2603                 dcb_config = 0;
2604         }
2605
2606         if (port_id_is_invalid(pid, ENABLED_WARN))
2607                 return;
2608
2609         printf("Stopping ports...\n");
2610
2611         RTE_ETH_FOREACH_DEV(pi) {
2612                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2613                         continue;
2614
2615                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2616                         printf("Please remove port %d from forwarding configuration.\n", pi);
2617                         continue;
2618                 }
2619
2620                 if (port_is_bonding_slave(pi)) {
2621                         printf("Please remove port %d from bonded device.\n", pi);
2622                         continue;
2623                 }
2624
2625                 port = &ports[pi];
2626                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2627                                                 RTE_PORT_HANDLING) == 0)
2628                         continue;
2629
2630                 rte_eth_dev_stop(pi);
2631
2632                 if (rte_atomic16_cmpset(&(port->port_status),
2633                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2634                         printf("Port %d can not be set into stopped\n", pi);
2635                 need_check_link_status = 1;
2636         }
2637         if (need_check_link_status && !no_link_check)
2638                 check_all_ports_link_status(RTE_PORT_ALL);
2639
2640         printf("Done\n");
2641 }
2642
2643 static void
2644 remove_invalid_ports_in(portid_t *array, portid_t *total)
2645 {
2646         portid_t i;
2647         portid_t new_total = 0;
2648
2649         for (i = 0; i < *total; i++)
2650                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2651                         array[new_total] = array[i];
2652                         new_total++;
2653                 }
2654         *total = new_total;
2655 }
2656
2657 static void
2658 remove_invalid_ports(void)
2659 {
2660         remove_invalid_ports_in(ports_ids, &nb_ports);
2661         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2662         nb_cfg_ports = nb_fwd_ports;
2663 }
2664
2665 void
2666 close_port(portid_t pid)
2667 {
2668         portid_t pi;
2669         struct rte_port *port;
2670
2671         if (port_id_is_invalid(pid, ENABLED_WARN))
2672                 return;
2673
2674         printf("Closing ports...\n");
2675
2676         RTE_ETH_FOREACH_DEV(pi) {
2677                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2678                         continue;
2679
2680                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2681                         printf("Please remove port %d from forwarding configuration.\n", pi);
2682                         continue;
2683                 }
2684
2685                 if (port_is_bonding_slave(pi)) {
2686                         printf("Please remove port %d from bonded device.\n", pi);
2687                         continue;
2688                 }
2689
2690                 port = &ports[pi];
2691                 if (rte_atomic16_cmpset(&(port->port_status),
2692                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2693                         printf("Port %d is already closed\n", pi);
2694                         continue;
2695                 }
2696
2697                 if (rte_atomic16_cmpset(&(port->port_status),
2698                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2699                         printf("Port %d is now not stopped\n", pi);
2700                         continue;
2701                 }
2702
2703                 if (port->flow_list)
2704                         port_flow_flush(pi);
2705                 rte_eth_dev_close(pi);
2706
2707                 remove_invalid_ports();
2708
2709                 if (rte_atomic16_cmpset(&(port->port_status),
2710                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2711                         printf("Port %d cannot be set to closed\n", pi);
2712         }
2713
2714         printf("Done\n");
2715 }
2716
2717 void
2718 reset_port(portid_t pid)
2719 {
2720         int diag;
2721         portid_t pi;
2722         struct rte_port *port;
2723
2724         if (port_id_is_invalid(pid, ENABLED_WARN))
2725                 return;
2726
2727         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2728                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2729                 printf("Can not reset port(s), please stop port(s) first.\n");
2730                 return;
2731         }
2732
2733         printf("Resetting ports...\n");
2734
2735         RTE_ETH_FOREACH_DEV(pi) {
2736                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2737                         continue;
2738
2739                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2740                         printf("Please remove port %d from forwarding "
2741                                "configuration.\n", pi);
2742                         continue;
2743                 }
2744
2745                 if (port_is_bonding_slave(pi)) {
2746                         printf("Please remove port %d from bonded device.\n",
2747                                pi);
2748                         continue;
2749                 }
2750
2751                 diag = rte_eth_dev_reset(pi);
2752                 if (diag == 0) {
2753                         port = &ports[pi];
2754                         port->need_reconfig = 1;
2755                         port->need_reconfig_queues = 1;
2756                 } else {
2757                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2758                 }
2759         }
2760
2761         printf("Done\n");
2762 }
2763
2764 void
2765 attach_port(char *identifier)
2766 {
2767         portid_t pi;
2768         struct rte_dev_iterator iterator;
2769
2770         printf("Attaching a new port...\n");
2771
2772         if (identifier == NULL) {
2773                 printf("Invalid parameters are specified\n");
2774                 return;
2775         }
2776
2777         if (rte_dev_probe(identifier) < 0) {
2778                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2779                 return;
2780         }
2781
2782         /* first attach mode: event */
2783         if (setup_on_probe_event) {
2784                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2785                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2786                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2787                                         ports[pi].need_setup != 0)
2788                                 setup_attached_port(pi);
2789                 return;
2790         }
2791
2792         /* second attach mode: iterator */
2793         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2794                 /* setup ports matching the devargs used for probing */
2795                 if (port_is_forwarding(pi))
2796                         continue; /* port was already attached before */
2797                 setup_attached_port(pi);
2798         }
2799 }
2800
2801 static void
2802 setup_attached_port(portid_t pi)
2803 {
2804         unsigned int socket_id;
2805         int ret;
2806
2807         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2808         /* if socket_id is invalid, set to the first available socket. */
2809         if (check_socket_id(socket_id) < 0)
2810                 socket_id = socket_ids[0];
2811         reconfig(pi, socket_id);
2812         ret = rte_eth_promiscuous_enable(pi);
2813         if (ret != 0)
2814                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2815                         pi, rte_strerror(-ret));
2816
2817         ports_ids[nb_ports++] = pi;
2818         fwd_ports_ids[nb_fwd_ports++] = pi;
2819         nb_cfg_ports = nb_fwd_ports;
2820         ports[pi].need_setup = 0;
2821         ports[pi].port_status = RTE_PORT_STOPPED;
2822
2823         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2824         printf("Done\n");
2825 }
2826
2827 static void
2828 detach_device(struct rte_device *dev)
2829 {
2830         portid_t sibling;
2831
2832         if (dev == NULL) {
2833                 printf("Device already removed\n");
2834                 return;
2835         }
2836
2837         printf("Removing a device...\n");
2838
2839         if (rte_dev_remove(dev) < 0) {
2840                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2841                 return;
2842         }
2843         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2844                 /* reset mapping between old ports and removed device */
2845                 rte_eth_devices[sibling].device = NULL;
2846                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2847                         /* sibling ports are forced to be closed */
2848                         ports[sibling].port_status = RTE_PORT_CLOSED;
2849                         printf("Port %u is closed\n", sibling);
2850                 }
2851         }
2852
2853         remove_invalid_ports();
2854
2855         printf("Device is detached\n");
2856         printf("Now total ports is %d\n", nb_ports);
2857         printf("Done\n");
2858         return;
2859 }
2860
2861 void
2862 detach_port_device(portid_t port_id)
2863 {
2864         if (port_id_is_invalid(port_id, ENABLED_WARN))
2865                 return;
2866
2867         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2868                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2869                         printf("Port not stopped\n");
2870                         return;
2871                 }
2872                 printf("Port was not closed\n");
2873                 if (ports[port_id].flow_list)
2874                         port_flow_flush(port_id);
2875         }
2876
2877         detach_device(rte_eth_devices[port_id].device);
2878 }
2879
2880 void
2881 detach_devargs(char *identifier)
2882 {
2883         struct rte_dev_iterator iterator;
2884         struct rte_devargs da;
2885         portid_t port_id;
2886
2887         printf("Removing a device...\n");
2888
2889         memset(&da, 0, sizeof(da));
2890         if (rte_devargs_parsef(&da, "%s", identifier)) {
2891                 printf("cannot parse identifier\n");
2892                 if (da.args)
2893                         free(da.args);
2894                 return;
2895         }
2896
2897         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2898                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2899                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2900                                 printf("Port %u not stopped\n", port_id);
2901                                 rte_eth_iterator_cleanup(&iterator);
2902                                 return;
2903                         }
2904
2905                         /* sibling ports are forced to be closed */
2906                         if (ports[port_id].flow_list)
2907                                 port_flow_flush(port_id);
2908                         ports[port_id].port_status = RTE_PORT_CLOSED;
2909                         printf("Port %u is now closed\n", port_id);
2910                 }
2911         }
2912
2913         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2914                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2915                             da.name, da.bus->name);
2916                 return;
2917         }
2918
2919         remove_invalid_ports();
2920
2921         printf("Device %s is detached\n", identifier);
2922         printf("Now total ports is %d\n", nb_ports);
2923         printf("Done\n");
2924 }
2925
2926 void
2927 pmd_test_exit(void)
2928 {
2929         portid_t pt_id;
2930         int ret;
2931         int i;
2932
2933         if (test_done == 0)
2934                 stop_packet_forwarding();
2935
2936         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2937                 if (mempools[i]) {
2938                         if (mp_alloc_type == MP_ALLOC_ANON)
2939                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2940                                                      NULL);
2941                 }
2942         }
2943         if (ports != NULL) {
2944                 no_link_check = 1;
2945                 RTE_ETH_FOREACH_DEV(pt_id) {
2946                         printf("\nStopping port %d...\n", pt_id);
2947                         fflush(stdout);
2948                         stop_port(pt_id);
2949                 }
2950                 RTE_ETH_FOREACH_DEV(pt_id) {
2951                         printf("\nShutting down port %d...\n", pt_id);
2952                         fflush(stdout);
2953                         close_port(pt_id);
2954                 }
2955         }
2956
2957         if (hot_plug) {
2958                 ret = rte_dev_event_monitor_stop();
2959                 if (ret) {
2960                         RTE_LOG(ERR, EAL,
2961                                 "fail to stop device event monitor.");
2962                         return;
2963                 }
2964
2965                 ret = rte_dev_event_callback_unregister(NULL,
2966                         dev_event_callback, NULL);
2967                 if (ret < 0) {
2968                         RTE_LOG(ERR, EAL,
2969                                 "fail to unregister device event callback.\n");
2970                         return;
2971                 }
2972
2973                 ret = rte_dev_hotplug_handle_disable();
2974                 if (ret) {
2975                         RTE_LOG(ERR, EAL,
2976                                 "fail to disable hotplug handling.\n");
2977                         return;
2978                 }
2979         }
2980         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2981                 if (mempools[i])
2982                         rte_mempool_free(mempools[i]);
2983         }
2984
2985         printf("\nBye...\n");
2986 }
2987
2988 typedef void (*cmd_func_t)(void);
2989 struct pmd_test_command {
2990         const char *cmd_name;
2991         cmd_func_t cmd_func;
2992 };
2993
2994 /* Check the link status of all ports in up to 9s, and print them finally */
2995 static void
2996 check_all_ports_link_status(uint32_t port_mask)
2997 {
2998 #define CHECK_INTERVAL 100 /* 100ms */
2999 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3000         portid_t portid;
3001         uint8_t count, all_ports_up, print_flag = 0;
3002         struct rte_eth_link link;
3003         int ret;
3004
3005         printf("Checking link statuses...\n");
3006         fflush(stdout);
3007         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3008                 all_ports_up = 1;
3009                 RTE_ETH_FOREACH_DEV(portid) {
3010                         if ((port_mask & (1 << portid)) == 0)
3011                                 continue;
3012                         memset(&link, 0, sizeof(link));
3013                         ret = rte_eth_link_get_nowait(portid, &link);
3014                         if (ret < 0) {
3015                                 all_ports_up = 0;
3016                                 if (print_flag == 1)
3017                                         printf("Port %u link get failed: %s\n",
3018                                                 portid, rte_strerror(-ret));
3019                                 continue;
3020                         }
3021                         /* print link status if flag set */
3022                         if (print_flag == 1) {
3023                                 if (link.link_status)
3024                                         printf(
3025                                         "Port%d Link Up. speed %u Mbps- %s\n",
3026                                         portid, link.link_speed,
3027                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3028                                         ("full-duplex") : ("half-duplex"));
3029                                 else
3030                                         printf("Port %d Link Down\n", portid);
3031                                 continue;
3032                         }
3033                         /* clear all_ports_up flag if any link down */
3034                         if (link.link_status == ETH_LINK_DOWN) {
3035                                 all_ports_up = 0;
3036                                 break;
3037                         }
3038                 }
3039                 /* after finally printing all link status, get out */
3040                 if (print_flag == 1)
3041                         break;
3042
3043                 if (all_ports_up == 0) {
3044                         fflush(stdout);
3045                         rte_delay_ms(CHECK_INTERVAL);
3046                 }
3047
3048                 /* set the print_flag if all ports up or timeout */
3049                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3050                         print_flag = 1;
3051                 }
3052
3053                 if (lsc_interrupt)
3054                         break;
3055         }
3056 }
3057
3058 /*
3059  * This callback is for remove a port for a device. It has limitation because
3060  * it is not for multiple port removal for a device.
3061  * TODO: the device detach invoke will plan to be removed from user side to
3062  * eal. And convert all PMDs to free port resources on ether device closing.
3063  */
3064 static void
3065 rmv_port_callback(void *arg)
3066 {
3067         int need_to_start = 0;
3068         int org_no_link_check = no_link_check;
3069         portid_t port_id = (intptr_t)arg;
3070         struct rte_device *dev;
3071
3072         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3073
3074         if (!test_done && port_is_forwarding(port_id)) {
3075                 need_to_start = 1;
3076                 stop_packet_forwarding();
3077         }
3078         no_link_check = 1;
3079         stop_port(port_id);
3080         no_link_check = org_no_link_check;
3081
3082         /* Save rte_device pointer before closing ethdev port */
3083         dev = rte_eth_devices[port_id].device;
3084         close_port(port_id);
3085         detach_device(dev); /* might be already removed or have more ports */
3086
3087         if (need_to_start)
3088                 start_packet_forwarding(0);
3089 }
3090
3091 /* This function is used by the interrupt thread */
3092 static int
3093 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3094                   void *ret_param)
3095 {
3096         RTE_SET_USED(param);
3097         RTE_SET_USED(ret_param);
3098
3099         if (type >= RTE_ETH_EVENT_MAX) {
3100                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3101                         port_id, __func__, type);
3102                 fflush(stderr);
3103         } else if (event_print_mask & (UINT32_C(1) << type)) {
3104                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3105                         eth_event_desc[type]);
3106                 fflush(stdout);
3107         }
3108
3109         switch (type) {
3110         case RTE_ETH_EVENT_NEW:
3111                 ports[port_id].need_setup = 1;
3112                 ports[port_id].port_status = RTE_PORT_HANDLING;
3113                 break;
3114         case RTE_ETH_EVENT_INTR_RMV:
3115                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3116                         break;
3117                 if (rte_eal_alarm_set(100000,
3118                                 rmv_port_callback, (void *)(intptr_t)port_id))
3119                         fprintf(stderr, "Could not set up deferred device removal\n");
3120                 break;
3121         default:
3122                 break;
3123         }
3124         return 0;
3125 }
3126
3127 static int
3128 register_eth_event_callback(void)
3129 {
3130         int ret;
3131         enum rte_eth_event_type event;
3132
3133         for (event = RTE_ETH_EVENT_UNKNOWN;
3134                         event < RTE_ETH_EVENT_MAX; event++) {
3135                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3136                                 event,
3137                                 eth_event_callback,
3138                                 NULL);
3139                 if (ret != 0) {
3140                         TESTPMD_LOG(ERR, "Failed to register callback for "
3141                                         "%s event\n", eth_event_desc[event]);
3142                         return -1;
3143                 }
3144         }
3145
3146         return 0;
3147 }
3148
3149 /* This function is used by the interrupt thread */
3150 static void
3151 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3152                              __rte_unused void *arg)
3153 {
3154         uint16_t port_id;
3155         int ret;
3156
3157         if (type >= RTE_DEV_EVENT_MAX) {
3158                 fprintf(stderr, "%s called upon invalid event %d\n",
3159                         __func__, type);
3160                 fflush(stderr);
3161         }
3162
3163         switch (type) {
3164         case RTE_DEV_EVENT_REMOVE:
3165                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3166                         device_name);
3167                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3168                 if (ret) {
3169                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3170                                 device_name);
3171                         return;
3172                 }
3173                 /*
3174                  * Because the user's callback is invoked in eal interrupt
3175                  * callback, the interrupt callback need to be finished before
3176                  * it can be unregistered when detaching device. So finish
3177                  * callback soon and use a deferred removal to detach device
3178                  * is need. It is a workaround, once the device detaching be
3179                  * moved into the eal in the future, the deferred removal could
3180                  * be deleted.
3181                  */
3182                 if (rte_eal_alarm_set(100000,
3183                                 rmv_port_callback, (void *)(intptr_t)port_id))
3184                         RTE_LOG(ERR, EAL,
3185                                 "Could not set up deferred device removal\n");
3186                 break;
3187         case RTE_DEV_EVENT_ADD:
3188                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3189                         device_name);
3190                 /* TODO: After finish kernel driver binding,
3191                  * begin to attach port.
3192                  */
3193                 break;
3194         default:
3195                 break;
3196         }
3197 }
3198
3199 static int
3200 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3201 {
3202         uint16_t i;
3203         int diag;
3204         uint8_t mapping_found = 0;
3205
3206         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3207                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3208                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3209                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3210                                         tx_queue_stats_mappings[i].queue_id,
3211                                         tx_queue_stats_mappings[i].stats_counter_id);
3212                         if (diag != 0)
3213                                 return diag;
3214                         mapping_found = 1;
3215                 }
3216         }
3217         if (mapping_found)
3218                 port->tx_queue_stats_mapping_enabled = 1;
3219         return 0;
3220 }
3221
3222 static int
3223 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3224 {
3225         uint16_t i;
3226         int diag;
3227         uint8_t mapping_found = 0;
3228
3229         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3230                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3231                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3232                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3233                                         rx_queue_stats_mappings[i].queue_id,
3234                                         rx_queue_stats_mappings[i].stats_counter_id);
3235                         if (diag != 0)
3236                                 return diag;
3237                         mapping_found = 1;
3238                 }
3239         }
3240         if (mapping_found)
3241                 port->rx_queue_stats_mapping_enabled = 1;
3242         return 0;
3243 }
3244
3245 static void
3246 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3247 {
3248         int diag = 0;
3249
3250         diag = set_tx_queue_stats_mapping_registers(pi, port);
3251         if (diag != 0) {
3252                 if (diag == -ENOTSUP) {
3253                         port->tx_queue_stats_mapping_enabled = 0;
3254                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3255                 }
3256                 else
3257                         rte_exit(EXIT_FAILURE,
3258                                         "set_tx_queue_stats_mapping_registers "
3259                                         "failed for port id=%d diag=%d\n",
3260                                         pi, diag);
3261         }
3262
3263         diag = set_rx_queue_stats_mapping_registers(pi, port);
3264         if (diag != 0) {
3265                 if (diag == -ENOTSUP) {
3266                         port->rx_queue_stats_mapping_enabled = 0;
3267                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3268                 }
3269                 else
3270                         rte_exit(EXIT_FAILURE,
3271                                         "set_rx_queue_stats_mapping_registers "
3272                                         "failed for port id=%d diag=%d\n",
3273                                         pi, diag);
3274         }
3275 }
3276
3277 static void
3278 rxtx_port_config(struct rte_port *port)
3279 {
3280         uint16_t qid;
3281         uint64_t offloads;
3282
3283         for (qid = 0; qid < nb_rxq; qid++) {
3284                 offloads = port->rx_conf[qid].offloads;
3285                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3286                 if (offloads != 0)
3287                         port->rx_conf[qid].offloads = offloads;
3288
3289                 /* Check if any Rx parameters have been passed */
3290                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3291                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3292
3293                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3294                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3295
3296                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3297                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3298
3299                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3300                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3301
3302                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3303                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3304
3305                 port->nb_rx_desc[qid] = nb_rxd;
3306         }
3307
3308         for (qid = 0; qid < nb_txq; qid++) {
3309                 offloads = port->tx_conf[qid].offloads;
3310                 port->tx_conf[qid] = port->dev_info.default_txconf;
3311                 if (offloads != 0)
3312                         port->tx_conf[qid].offloads = offloads;
3313
3314                 /* Check if any Tx parameters have been passed */
3315                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3316                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3317
3318                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3319                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3320
3321                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3322                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3323
3324                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3325                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3326
3327                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3328                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3329
3330                 port->nb_tx_desc[qid] = nb_txd;
3331         }
3332 }
3333
3334 void
3335 init_port_config(void)
3336 {
3337         portid_t pid;
3338         struct rte_port *port;
3339         int ret;
3340
3341         RTE_ETH_FOREACH_DEV(pid) {
3342                 port = &ports[pid];
3343                 port->dev_conf.fdir_conf = fdir_conf;
3344
3345                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3346                 if (ret != 0)
3347                         return;
3348
3349                 if (nb_rxq > 1) {
3350                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3351                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3352                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3353                 } else {
3354                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3355                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3356                 }
3357
3358                 if (port->dcb_flag == 0) {
3359                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3360                                 port->dev_conf.rxmode.mq_mode =
3361                                         (enum rte_eth_rx_mq_mode)
3362                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3363                         else
3364                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3365                 }
3366
3367                 rxtx_port_config(port);
3368
3369                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3370                 if (ret != 0)
3371                         return;
3372
3373                 map_port_queue_stats_mapping_registers(pid, port);
3374 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3375                 rte_pmd_ixgbe_bypass_init(pid);
3376 #endif
3377
3378                 if (lsc_interrupt &&
3379                     (rte_eth_devices[pid].data->dev_flags &
3380                      RTE_ETH_DEV_INTR_LSC))
3381                         port->dev_conf.intr_conf.lsc = 1;
3382                 if (rmv_interrupt &&
3383                     (rte_eth_devices[pid].data->dev_flags &
3384                      RTE_ETH_DEV_INTR_RMV))
3385                         port->dev_conf.intr_conf.rmv = 1;
3386         }
3387 }
3388
3389 void set_port_slave_flag(portid_t slave_pid)
3390 {
3391         struct rte_port *port;
3392
3393         port = &ports[slave_pid];
3394         port->slave_flag = 1;
3395 }
3396
3397 void clear_port_slave_flag(portid_t slave_pid)
3398 {
3399         struct rte_port *port;
3400
3401         port = &ports[slave_pid];
3402         port->slave_flag = 0;
3403 }
3404
3405 uint8_t port_is_bonding_slave(portid_t slave_pid)
3406 {
3407         struct rte_port *port;
3408
3409         port = &ports[slave_pid];
3410         if ((rte_eth_devices[slave_pid].data->dev_flags &
3411             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3412                 return 1;
3413         return 0;
3414 }
3415
3416 const uint16_t vlan_tags[] = {
3417                 0,  1,  2,  3,  4,  5,  6,  7,
3418                 8,  9, 10, 11,  12, 13, 14, 15,
3419                 16, 17, 18, 19, 20, 21, 22, 23,
3420                 24, 25, 26, 27, 28, 29, 30, 31
3421 };
3422
3423 static  int
3424 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3425                  enum dcb_mode_enable dcb_mode,
3426                  enum rte_eth_nb_tcs num_tcs,
3427                  uint8_t pfc_en)
3428 {
3429         uint8_t i;
3430         int32_t rc;
3431         struct rte_eth_rss_conf rss_conf;
3432
3433         /*
3434          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3435          * given above, and the number of traffic classes available for use.
3436          */
3437         if (dcb_mode == DCB_VT_ENABLED) {
3438                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3439                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3440                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3441                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3442
3443                 /* VMDQ+DCB RX and TX configurations */
3444                 vmdq_rx_conf->enable_default_pool = 0;
3445                 vmdq_rx_conf->default_pool = 0;
3446                 vmdq_rx_conf->nb_queue_pools =
3447                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3448                 vmdq_tx_conf->nb_queue_pools =
3449                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3450
3451                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3452                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3453                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3454                         vmdq_rx_conf->pool_map[i].pools =
3455                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3456                 }
3457                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3458                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3459                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3460                 }
3461
3462                 /* set DCB mode of RX and TX of multiple queues */
3463                 eth_conf->rxmode.mq_mode =
3464                                 (enum rte_eth_rx_mq_mode)
3465                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3466                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3467         } else {
3468                 struct rte_eth_dcb_rx_conf *rx_conf =
3469                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3470                 struct rte_eth_dcb_tx_conf *tx_conf =
3471                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3472
3473                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3474
3475                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3476                 if (rc != 0)
3477                         return rc;
3478
3479                 rx_conf->nb_tcs = num_tcs;
3480                 tx_conf->nb_tcs = num_tcs;
3481
3482                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3483                         rx_conf->dcb_tc[i] = i % num_tcs;
3484                         tx_conf->dcb_tc[i] = i % num_tcs;
3485                 }
3486
3487                 eth_conf->rxmode.mq_mode =
3488                                 (enum rte_eth_rx_mq_mode)
3489                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3490                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3491                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3492         }
3493
3494         if (pfc_en)
3495                 eth_conf->dcb_capability_en =
3496                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3497         else
3498                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3499
3500         return 0;
3501 }
3502
3503 int
3504 init_port_dcb_config(portid_t pid,
3505                      enum dcb_mode_enable dcb_mode,
3506                      enum rte_eth_nb_tcs num_tcs,
3507                      uint8_t pfc_en)
3508 {
3509         struct rte_eth_conf port_conf;
3510         struct rte_port *rte_port;
3511         int retval;
3512         uint16_t i;
3513
3514         rte_port = &ports[pid];
3515
3516         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3517         /* Enter DCB configuration status */
3518         dcb_config = 1;
3519
3520         port_conf.rxmode = rte_port->dev_conf.rxmode;
3521         port_conf.txmode = rte_port->dev_conf.txmode;
3522
3523         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3524         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3525         if (retval < 0)
3526                 return retval;
3527         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3528
3529         /* re-configure the device . */
3530         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3531         if (retval < 0)
3532                 return retval;
3533
3534         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3535         if (retval != 0)
3536                 return retval;
3537
3538         /* If dev_info.vmdq_pool_base is greater than 0,
3539          * the queue id of vmdq pools is started after pf queues.
3540          */
3541         if (dcb_mode == DCB_VT_ENABLED &&
3542             rte_port->dev_info.vmdq_pool_base > 0) {
3543                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3544                         " for port %d.", pid);
3545                 return -1;
3546         }
3547
3548         /* Assume the ports in testpmd have the same dcb capability
3549          * and has the same number of rxq and txq in dcb mode
3550          */
3551         if (dcb_mode == DCB_VT_ENABLED) {
3552                 if (rte_port->dev_info.max_vfs > 0) {
3553                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3554                         nb_txq = rte_port->dev_info.nb_tx_queues;
3555                 } else {
3556                         nb_rxq = rte_port->dev_info.max_rx_queues;
3557                         nb_txq = rte_port->dev_info.max_tx_queues;
3558                 }
3559         } else {
3560                 /*if vt is disabled, use all pf queues */
3561                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3562                         nb_rxq = rte_port->dev_info.max_rx_queues;
3563                         nb_txq = rte_port->dev_info.max_tx_queues;
3564                 } else {
3565                         nb_rxq = (queueid_t)num_tcs;
3566                         nb_txq = (queueid_t)num_tcs;
3567
3568                 }
3569         }
3570         rx_free_thresh = 64;
3571
3572         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3573
3574         rxtx_port_config(rte_port);
3575         /* VLAN filter */
3576         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3577         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3578                 rx_vft_set(pid, vlan_tags[i], 1);
3579
3580         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3581         if (retval != 0)
3582                 return retval;
3583
3584         map_port_queue_stats_mapping_registers(pid, rte_port);
3585
3586         rte_port->dcb_flag = 1;
3587
3588         return 0;
3589 }
3590
3591 static void
3592 init_port(void)
3593 {
3594         /* Configuration of Ethernet ports. */
3595         ports = rte_zmalloc("testpmd: ports",
3596                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3597                             RTE_CACHE_LINE_SIZE);
3598         if (ports == NULL) {
3599                 rte_exit(EXIT_FAILURE,
3600                                 "rte_zmalloc(%d struct rte_port) failed\n",
3601                                 RTE_MAX_ETHPORTS);
3602         }
3603
3604         /* Initialize ports NUMA structures */
3605         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3606         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3607         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3608 }
3609
3610 static void
3611 force_quit(void)
3612 {
3613         pmd_test_exit();
3614         prompt_exit();
3615 }
3616
3617 static void
3618 print_stats(void)
3619 {
3620         uint8_t i;
3621         const char clr[] = { 27, '[', '2', 'J', '\0' };
3622         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3623
3624         /* Clear screen and move to top left */
3625         printf("%s%s", clr, top_left);
3626
3627         printf("\nPort statistics ====================================");
3628         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3629                 nic_stats_display(fwd_ports_ids[i]);
3630
3631         fflush(stdout);
3632 }
3633
3634 static void
3635 signal_handler(int signum)
3636 {
3637         if (signum == SIGINT || signum == SIGTERM) {
3638                 printf("\nSignal %d received, preparing to exit...\n",
3639                                 signum);
3640 #ifdef RTE_LIBRTE_PDUMP
3641                 /* uninitialize packet capture framework */
3642                 rte_pdump_uninit();
3643 #endif
3644 #ifdef RTE_LIBRTE_LATENCY_STATS
3645                 if (latencystats_enabled != 0)
3646                         rte_latencystats_uninit();
3647 #endif
3648                 force_quit();
3649                 /* Set flag to indicate the force termination. */
3650                 f_quit = 1;
3651                 /* exit with the expected status */
3652                 signal(signum, SIG_DFL);
3653                 kill(getpid(), signum);
3654         }
3655 }
3656
3657 int
3658 main(int argc, char** argv)
3659 {
3660         int diag;
3661         portid_t port_id;
3662         uint16_t count;
3663         int ret;
3664
3665         signal(SIGINT, signal_handler);
3666         signal(SIGTERM, signal_handler);
3667
3668         testpmd_logtype = rte_log_register("testpmd");
3669         if (testpmd_logtype < 0)
3670                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3671         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3672
3673         diag = rte_eal_init(argc, argv);
3674         if (diag < 0)
3675                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3676                          rte_strerror(rte_errno));
3677
3678         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3679                 rte_exit(EXIT_FAILURE,
3680                          "Secondary process type not supported.\n");
3681
3682         ret = register_eth_event_callback();
3683         if (ret != 0)
3684                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3685
3686 #ifdef RTE_LIBRTE_PDUMP
3687         /* initialize packet capture framework */
3688         rte_pdump_init();
3689 #endif
3690
3691         count = 0;
3692         RTE_ETH_FOREACH_DEV(port_id) {
3693                 ports_ids[count] = port_id;
3694                 count++;
3695         }
3696         nb_ports = (portid_t) count;
3697         if (nb_ports == 0)
3698                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3699
3700         /* allocate port structures, and init them */
3701         init_port();
3702
3703         set_def_fwd_config();
3704         if (nb_lcores == 0)
3705                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3706                          "Check the core mask argument\n");
3707
3708         /* Bitrate/latency stats disabled by default */
3709 #ifdef RTE_LIBRTE_BITRATE
3710         bitrate_enabled = 0;
3711 #endif
3712 #ifdef RTE_LIBRTE_LATENCY_STATS
3713         latencystats_enabled = 0;
3714 #endif
3715
3716         /* on FreeBSD, mlockall() is disabled by default */
3717 #ifdef RTE_EXEC_ENV_FREEBSD
3718         do_mlockall = 0;
3719 #else
3720         do_mlockall = 1;
3721 #endif
3722
3723         argc -= diag;
3724         argv += diag;
3725         if (argc > 1)
3726                 launch_args_parse(argc, argv);
3727
3728         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3729                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3730                         strerror(errno));
3731         }
3732
3733         if (tx_first && interactive)
3734                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3735                                 "interactive mode.\n");
3736
3737         if (tx_first && lsc_interrupt) {
3738                 printf("Warning: lsc_interrupt needs to be off when "
3739                                 " using tx_first. Disabling.\n");
3740                 lsc_interrupt = 0;
3741         }
3742
3743         if (!nb_rxq && !nb_txq)
3744                 printf("Warning: Either rx or tx queues should be non-zero\n");
3745
3746         if (nb_rxq > 1 && nb_rxq > nb_txq)
3747                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3748                        "but nb_txq=%d will prevent to fully test it.\n",
3749                        nb_rxq, nb_txq);
3750
3751         init_config();
3752
3753         if (hot_plug) {
3754                 ret = rte_dev_hotplug_handle_enable();
3755                 if (ret) {
3756                         RTE_LOG(ERR, EAL,
3757                                 "fail to enable hotplug handling.");
3758                         return -1;
3759                 }
3760
3761                 ret = rte_dev_event_monitor_start();
3762                 if (ret) {
3763                         RTE_LOG(ERR, EAL,
3764                                 "fail to start device event monitoring.");
3765                         return -1;
3766                 }
3767
3768                 ret = rte_dev_event_callback_register(NULL,
3769                         dev_event_callback, NULL);
3770                 if (ret) {
3771                         RTE_LOG(ERR, EAL,
3772                                 "fail  to register device event callback\n");
3773                         return -1;
3774                 }
3775         }
3776
3777         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3778                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3779
3780         /* set all ports to promiscuous mode by default */
3781         RTE_ETH_FOREACH_DEV(port_id) {
3782                 ret = rte_eth_promiscuous_enable(port_id);
3783                 if (ret != 0)
3784                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3785                                 port_id, rte_strerror(-ret));
3786         }
3787
3788         /* Init metrics library */
3789         rte_metrics_init(rte_socket_id());
3790
3791 #ifdef RTE_LIBRTE_LATENCY_STATS
3792         if (latencystats_enabled != 0) {
3793                 int ret = rte_latencystats_init(1, NULL);
3794                 if (ret)
3795                         printf("Warning: latencystats init()"
3796                                 " returned error %d\n", ret);
3797                 printf("Latencystats running on lcore %d\n",
3798                         latencystats_lcore_id);
3799         }
3800 #endif
3801
3802         /* Setup bitrate stats */
3803 #ifdef RTE_LIBRTE_BITRATE
3804         if (bitrate_enabled != 0) {
3805                 bitrate_data = rte_stats_bitrate_create();
3806                 if (bitrate_data == NULL)
3807                         rte_exit(EXIT_FAILURE,
3808                                 "Could not allocate bitrate data.\n");
3809                 rte_stats_bitrate_reg(bitrate_data);
3810         }
3811 #endif
3812
3813 #ifdef RTE_LIBRTE_CMDLINE
3814         if (strlen(cmdline_filename) != 0)
3815                 cmdline_read_from_file(cmdline_filename);
3816
3817         if (interactive == 1) {
3818                 if (auto_start) {
3819                         printf("Start automatic packet forwarding\n");
3820                         start_packet_forwarding(0);
3821                 }
3822                 prompt();
3823                 pmd_test_exit();
3824         } else
3825 #endif
3826         {
3827                 char c;
3828                 int rc;
3829
3830                 f_quit = 0;
3831
3832                 printf("No commandline core given, start packet forwarding\n");
3833                 start_packet_forwarding(tx_first);
3834                 if (stats_period != 0) {
3835                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3836                         uint64_t timer_period;
3837
3838                         /* Convert to number of cycles */
3839                         timer_period = stats_period * rte_get_timer_hz();
3840
3841                         while (f_quit == 0) {
3842                                 cur_time = rte_get_timer_cycles();
3843                                 diff_time += cur_time - prev_time;
3844
3845                                 if (diff_time >= timer_period) {
3846                                         print_stats();
3847                                         /* Reset the timer */
3848                                         diff_time = 0;
3849                                 }
3850                                 /* Sleep to avoid unnecessary checks */
3851                                 prev_time = cur_time;
3852                                 sleep(1);
3853                         }
3854                 }
3855
3856                 printf("Press enter to exit\n");
3857                 rc = read(0, &c, 1);
3858                 pmd_test_exit();
3859                 if (rc < 0)
3860                         return 1;
3861         }
3862
3863         ret = rte_eal_cleanup();
3864         if (ret != 0)
3865                 rte_exit(EXIT_FAILURE,
3866                          "EAL cleanup failed: %s\n", strerror(-ret));
3867
3868         return EXIT_SUCCESS;
3869 }