app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 199 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 200                                       * specified on command-line. */
 201 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 202
 203 /*
 204  * In container, it cannot terminate the process which running with 'stats-period'
 205  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 206  */
 207 uint8_t f_quit;
 208
 209 /*
 210  * Configuration of packet segments used by the "txonly" processing engine.
 211  */
 212 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 213 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 214         TXONLY_DEF_PACKET_LEN,
 215 };
 216 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 217
 218 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 219 /**< Split policy for packets to TX. */
 220
 221 uint8_t txonly_multi_flow;
 222 /**< Whether multiple flows are generated in TXONLY mode. */
 223
 224 uint32_t tx_pkt_times_inter;
 225 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 226
 227 uint32_t tx_pkt_times_intra;
 228 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 229
 230 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 231 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 232
 233 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 234 uint8_t dcb_config = 0;
 235
 236 /* Whether the dcb is in testing status */
 237 uint8_t dcb_test = 0;
 238
 239 /*
 240  * Configurable number of RX/TX queues.
 241  */
 242 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 243 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 244 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 245
 246 /*
 247  * Configurable number of RX/TX ring descriptors.
 248  * Defaults are supplied by drivers via ethdev.
 249  */
 250 #define RTE_TEST_RX_DESC_DEFAULT 0
 251 #define RTE_TEST_TX_DESC_DEFAULT 0
 252 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 253 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 254
 255 #define RTE_PMD_PARAM_UNSET -1
 256 /*
 257  * Configurable values of RX and TX ring threshold registers.
 258  */
 259
 260 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 262 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 263
 264 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 265 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 266 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX free threshold.
 270  */
 271 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of RX drop enable.
 275  */
 276 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX free threshold.
 280  */
 281 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of TX RS bit threshold.
 285  */
 286 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of buffered packets before sending.
 290  */
 291 uint16_t noisy_tx_sw_bufsz;
 292
 293 /*
 294  * Configurable value of packet buffer timeout.
 295  */
 296 uint16_t noisy_tx_sw_buf_flush_time;
 297
 298 /*
 299  * Configurable value for size of VNF internal memory area
 300  * used for simulating noisy neighbour behaviour
 301  */
 302 uint64_t noisy_lkup_mem_sz;
 303
 304 /*
 305  * Configurable value of number of random writes done in
 306  * VNF simulation memory area.
 307  */
 308 uint64_t noisy_lkup_num_writes;
 309
 310 /*
 311  * Configurable value of number of random reads done in
 312  * VNF simulation memory area.
 313  */
 314 uint64_t noisy_lkup_num_reads;
 315
 316 /*
 317  * Configurable value of number of random reads/writes done in
 318  * VNF simulation memory area.
 319  */
 320 uint64_t noisy_lkup_num_reads_writes;
 321
 322 /*
 323  * Receive Side Scaling (RSS) configuration.
 324  */
 325 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 326
 327 /*
 328  * Port topology configuration
 329  */
 330 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 331
 332 /*
 333  * Avoids to flush all the RX streams before starts forwarding.
 334  */
 335 uint8_t no_flush_rx = 0; /* flush by default */
 336
 337 /*
 338  * Flow API isolated mode.
 339  */
 340 uint8_t flow_isolate_all;
 341
 342 /*
 343  * Avoids to check link status when starting/stopping a port.
 344  */
 345 uint8_t no_link_check = 0; /* check by default */
 346
 347 /*
 348  * Don't automatically start all ports in interactive mode.
 349  */
 350 uint8_t no_device_start = 0;
 351
 352 /*
 353  * Enable link status change notification
 354  */
 355 uint8_t lsc_interrupt = 1; /* enabled by default */
 356
 357 /*
 358  * Enable device removal notification.
 359  */
 360 uint8_t rmv_interrupt = 1; /* enabled by default */
 361
 362 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 363
 364 /* After attach, port setup is called on event or by iterator */
 365 bool setup_on_probe_event = true;
 366
 367 /* Clear ptypes on port initialization. */
 368 uint8_t clear_ptypes = true;
 369
 370 /* Pretty printing of ethdev events */
 371 static const char * const eth_event_desc[] = {
 372         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 373         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 374         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 375         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 376         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 377         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 378         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 379         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 380         [RTE_ETH_EVENT_NEW] = "device probed",
 381         [RTE_ETH_EVENT_DESTROY] = "device released",
 382         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 383         [RTE_ETH_EVENT_MAX] = NULL,
 384 };
 385
 386 /*
 387  * Display or mask ether events
 388  * Default to all events except VF_MBOX
 389  */
 390 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 391                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 392                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 393                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 394                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 395                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 396                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 397                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 398 /*
 399  * Decide if all memory are locked for performance.
 400  */
 401 int do_mlockall = 0;
 402
 403 /*
 404  * NIC bypass mode configuration options.
 405  */
 406
 407 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 408 /* The NIC bypass watchdog timeout. */
 409 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 410 #endif
 411
 412
 413 #ifdef RTE_LIBRTE_LATENCY_STATS
 414
 415 /*
 416  * Set when latency stats is enabled in the commandline
 417  */
 418 uint8_t latencystats_enabled;
 419
 420 /*
 421  * Lcore ID to serive latency statistics.
 422  */
 423 lcoreid_t latencystats_lcore_id = -1;
 424
 425 #endif
 426
 427 /*
 428  * Ethernet device configuration.
 429  */
 430 struct rte_eth_rxmode rx_mode = {
 431         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 432                 /**< Default maximum frame length. */
 433 };
 434
 435 struct rte_eth_txmode tx_mode = {
 436         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 437 };
 438
 439 struct rte_fdir_conf fdir_conf = {
 440         .mode = RTE_FDIR_MODE_NONE,
 441         .pballoc = RTE_FDIR_PBALLOC_64K,
 442         .status = RTE_FDIR_REPORT_STATUS,
 443         .mask = {
 444                 .vlan_tci_mask = 0xFFEF,
 445                 .ipv4_mask     = {
 446                         .src_ip = 0xFFFFFFFF,
 447                         .dst_ip = 0xFFFFFFFF,
 448                 },
 449                 .ipv6_mask     = {
 450                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 451                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 452                 },
 453                 .src_port_mask = 0xFFFF,
 454                 .dst_port_mask = 0xFFFF,
 455                 .mac_addr_byte_mask = 0xFF,
 456                 .tunnel_type_mask = 1,
 457                 .tunnel_id_mask = 0xFFFFFFFF,
 458         },
 459         .drop_queue = 127,
 460 };
 461
 462 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 463
 464 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 465 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 466
 467 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 468 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 469
 470 uint16_t nb_tx_queue_stats_mappings = 0;
 471 uint16_t nb_rx_queue_stats_mappings = 0;
 472
 473 /*
 474  * Display zero values by default for xstats
 475  */
 476 uint8_t xstats_hide_zero;
 477
 478 /*
 479  * Measure of CPU cycles disabled by default
 480  */
 481 uint8_t record_core_cycles;
 482
 483 /*
 484  * Display of RX and TX bursts disabled by default
 485  */
 486 uint8_t record_burst_stats;
 487
 488 unsigned int num_sockets = 0;
 489 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 490
 491 #ifdef RTE_LIBRTE_BITRATESTATS
 492 /* Bitrate statistics */
 493 struct rte_stats_bitrates *bitrate_data;
 494 lcoreid_t bitrate_lcore_id;
 495 uint8_t bitrate_enabled;
 496 #endif
 497
 498 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 499 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 500
 501 /*
 502  * hexadecimal bitmask of RX mq mode can be enabled.
 503  */
 504 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 505
 506 /* Forward function declarations */
 507 static void setup_attached_port(portid_t pi);
 508 static void map_port_queue_stats_mapping_registers(portid_t pi,
 509                                                    struct rte_port *port);
 510 static void check_all_ports_link_status(uint32_t port_mask);
 511 static int eth_event_callback(portid_t port_id,
 512                               enum rte_eth_event_type type,
 513                               void *param, void *ret_param);
 514 static void dev_event_callback(const char *device_name,
 515                                 enum rte_dev_event_type type,
 516                                 void *param);
 517
 518 /*
 519  * Check if all the ports are started.
 520  * If yes, return positive value. If not, return zero.
 521  */
 522 static int all_ports_started(void);
 523
 524 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 525 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 526
 527 /* Holds the registered mbuf dynamic flags names. */
 528 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 529
 530 /*
 531  * Helper function to check if socket is already discovered.
 532  * If yes, return positive value. If not, return zero.
 533  */
 534 int
 535 new_socket_id(unsigned int socket_id)
 536 {
 537         unsigned int i;
 538
 539         for (i = 0; i < num_sockets; i++) {
 540                 if (socket_ids[i] == socket_id)
 541                         return 0;
 542         }
 543         return 1;
 544 }
 545
 546 /*
 547  * Setup default configuration.
 548  */
 549 static void
 550 set_default_fwd_lcores_config(void)
 551 {
 552         unsigned int i;
 553         unsigned int nb_lc;
 554         unsigned int sock_num;
 555
 556         nb_lc = 0;
 557         for (i = 0; i < RTE_MAX_LCORE; i++) {
 558                 if (!rte_lcore_is_enabled(i))
 559                         continue;
 560                 sock_num = rte_lcore_to_socket_id(i);
 561                 if (new_socket_id(sock_num)) {
 562                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 563                                 rte_exit(EXIT_FAILURE,
 564                                          "Total sockets greater than %u\n",
 565                                          RTE_MAX_NUMA_NODES);
 566                         }
 567                         socket_ids[num_sockets++] = sock_num;
 568                 }
 569                 if (i == rte_get_master_lcore())
 570                         continue;
 571                 fwd_lcores_cpuids[nb_lc++] = i;
 572         }
 573         nb_lcores = (lcoreid_t) nb_lc;
 574         nb_cfg_lcores = nb_lcores;
 575         nb_fwd_lcores = 1;
 576 }
 577
 578 static void
 579 set_def_peer_eth_addrs(void)
 580 {
 581         portid_t i;
 582
 583         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 584                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 585                 peer_eth_addrs[i].addr_bytes[5] = i;
 586         }
 587 }
 588
 589 static void
 590 set_default_fwd_ports_config(void)
 591 {
 592         portid_t pt_id;
 593         int i = 0;
 594
 595         RTE_ETH_FOREACH_DEV(pt_id) {
 596                 fwd_ports_ids[i++] = pt_id;
 597
 598                 /* Update sockets info according to the attached device */
 599                 int socket_id = rte_eth_dev_socket_id(pt_id);
 600                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 601                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 602                                 rte_exit(EXIT_FAILURE,
 603                                          "Total sockets greater than %u\n",
 604                                          RTE_MAX_NUMA_NODES);
 605                         }
 606                         socket_ids[num_sockets++] = socket_id;
 607                 }
 608         }
 609
 610         nb_cfg_ports = nb_ports;
 611         nb_fwd_ports = nb_ports;
 612 }
 613
 614 void
 615 set_def_fwd_config(void)
 616 {
 617         set_default_fwd_lcores_config();
 618         set_def_peer_eth_addrs();
 619         set_default_fwd_ports_config();
 620 }
 621
 622 /* extremely pessimistic estimation of memory required to create a mempool */
 623 static int
 624 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 625 {
 626         unsigned int n_pages, mbuf_per_pg, leftover;
 627         uint64_t total_mem, mbuf_mem, obj_sz;
 628
 629         /* there is no good way to predict how much space the mempool will
 630          * occupy because it will allocate chunks on the fly, and some of those
 631          * will come from default DPDK memory while some will come from our
 632          * external memory, so just assume 128MB will be enough for everyone.
 633          */
 634         uint64_t hdr_mem = 128 << 20;
 635
 636         /* account for possible non-contiguousness */
 637         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 638         if (obj_sz > pgsz) {
 639                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 640                 return -1;
 641         }
 642
 643         mbuf_per_pg = pgsz / obj_sz;
 644         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 645         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 646
 647         mbuf_mem = n_pages * pgsz;
 648
 649         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 650
 651         if (total_mem > SIZE_MAX) {
 652                 TESTPMD_LOG(ERR, "Memory size too big\n");
 653                 return -1;
 654         }
 655         *out = (size_t)total_mem;
 656
 657         return 0;
 658 }
 659
 660 static int
 661 pagesz_flags(uint64_t page_sz)
 662 {
 663         /* as per mmap() manpage, all page sizes are log2 of page size
 664          * shifted by MAP_HUGE_SHIFT
 665          */
 666         int log2 = rte_log2_u64(page_sz);
 667
 668         return (log2 << HUGE_SHIFT);
 669 }
 670
 671 static void *
 672 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 673 {
 674         void *addr;
 675         int flags;
 676
 677         /* allocate anonymous hugepages */
 678         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 679         if (huge)
 680                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 681
 682         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 683         if (addr == MAP_FAILED)
 684                 return NULL;
 685
 686         return addr;
 687 }
 688
 689 struct extmem_param {
 690         void *addr;
 691         size_t len;
 692         size_t pgsz;
 693         rte_iova_t *iova_table;
 694         unsigned int iova_table_len;
 695 };
 696
 697 static int
 698 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 699                 bool huge)
 700 {
 701         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 702                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 703         unsigned int cur_page, n_pages, pgsz_idx;
 704         size_t mem_sz, cur_pgsz;
 705         rte_iova_t *iovas = NULL;
 706         void *addr;
 707         int ret;
 708
 709         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 710                 /* skip anything that is too big */
 711                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 712                         continue;
 713
 714                 cur_pgsz = pgsizes[pgsz_idx];
 715
 716                 /* if we were told not to allocate hugepages, override */
 717                 if (!huge)
 718                         cur_pgsz = sysconf(_SC_PAGESIZE);
 719
 720                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 721                 if (ret < 0) {
 722                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 723                         return -1;
 724                 }
 725
 726                 /* allocate our memory */
 727                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 728
 729                 /* if we couldn't allocate memory with a specified page size,
 730                  * that doesn't mean we can't do it with other page sizes, so
 731                  * try another one.
 732                  */
 733                 if (addr == NULL)
 734                         continue;
 735
 736                 /* store IOVA addresses for every page in this memory area */
 737                 n_pages = mem_sz / cur_pgsz;
 738
 739                 iovas = malloc(sizeof(*iovas) * n_pages);
 740
 741                 if (iovas == NULL) {
 742                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 743                         goto fail;
 744                 }
 745                 /* lock memory if it's not huge pages */
 746                 if (!huge)
 747                         mlock(addr, mem_sz);
 748
 749                 /* populate IOVA addresses */
 750                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 751                         rte_iova_t iova;
 752                         size_t offset;
 753                         void *cur;
 754
 755                         offset = cur_pgsz * cur_page;
 756                         cur = RTE_PTR_ADD(addr, offset);
 757
 758                         /* touch the page before getting its IOVA */
 759                         *(volatile char *)cur = 0;
 760
 761                         iova = rte_mem_virt2iova(cur);
 762
 763                         iovas[cur_page] = iova;
 764                 }
 765
 766                 break;
 767         }
 768         /* if we couldn't allocate anything */
 769         if (iovas == NULL)
 770                 return -1;
 771
 772         param->addr = addr;
 773         param->len = mem_sz;
 774         param->pgsz = cur_pgsz;
 775         param->iova_table = iovas;
 776         param->iova_table_len = n_pages;
 777
 778         return 0;
 779 fail:
 780         if (iovas)
 781                 free(iovas);
 782         if (addr)
 783                 munmap(addr, mem_sz);
 784
 785         return -1;
 786 }
 787
 788 static int
 789 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 790 {
 791         struct extmem_param param;
 792         int socket_id, ret;
 793
 794         memset(&param, 0, sizeof(param));
 795
 796         /* check if our heap exists */
 797         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 798         if (socket_id < 0) {
 799                 /* create our heap */
 800                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 801                 if (ret < 0) {
 802                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 803                         return -1;
 804                 }
 805         }
 806
 807         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 808         if (ret < 0) {
 809                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 810                 return -1;
 811         }
 812
 813         /* we now have a valid memory area, so add it to heap */
 814         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 815                         param.addr, param.len, param.iova_table,
 816                         param.iova_table_len, param.pgsz);
 817
 818         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 819
 820         /* not needed any more */
 821         free(param.iova_table);
 822
 823         if (ret < 0) {
 824                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 825                 munmap(param.addr, param.len);
 826                 return -1;
 827         }
 828
 829         /* success */
 830
 831         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 832                         param.len >> 20);
 833
 834         return 0;
 835 }
 836 static void
 837 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 838              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 839 {
 840         uint16_t pid = 0;
 841         int ret;
 842
 843         RTE_ETH_FOREACH_DEV(pid) {
 844                 struct rte_eth_dev *dev =
 845                         &rte_eth_devices[pid];
 846
 847                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 848                                         memhdr->len);
 849                 if (ret) {
 850                         TESTPMD_LOG(DEBUG,
 851                                     "unable to DMA unmap addr 0x%p "
 852                                     "for device %s\n",
 853                                     memhdr->addr, dev->data->name);
 854                 }
 855         }
 856         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 857         if (ret) {
 858                 TESTPMD_LOG(DEBUG,
 859                             "unable to un-register addr 0x%p\n", memhdr->addr);
 860         }
 861 }
 862
 863 static void
 864 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 865            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 866 {
 867         uint16_t pid = 0;
 868         size_t page_size = sysconf(_SC_PAGESIZE);
 869         int ret;
 870
 871         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 872                                   page_size);
 873         if (ret) {
 874                 TESTPMD_LOG(DEBUG,
 875                             "unable to register addr 0x%p\n", memhdr->addr);
 876                 return;
 877         }
 878         RTE_ETH_FOREACH_DEV(pid) {
 879                 struct rte_eth_dev *dev =
 880                         &rte_eth_devices[pid];
 881
 882                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 883                                       memhdr->len);
 884                 if (ret) {
 885                         TESTPMD_LOG(DEBUG,
 886                                     "unable to DMA map addr 0x%p "
 887                                     "for device %s\n",
 888                                     memhdr->addr, dev->data->name);
 889                 }
 890         }
 891 }
 892
 893 static unsigned int
 894 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 895             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 896 {
 897         struct rte_pktmbuf_extmem *xmem;
 898         unsigned int ext_num, zone_num, elt_num;
 899         uint16_t elt_size;
 900
 901         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 902         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 903         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 904
 905         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 906         if (xmem == NULL) {
 907                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 908                                  "external buffer descriptors\n");
 909                 *ext_mem = NULL;
 910                 return 0;
 911         }
 912         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 913                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 914                 const struct rte_memzone *mz;
 915                 char mz_name[RTE_MEMZONE_NAMESIZE];
 916                 int ret;
 917
 918                 ret = snprintf(mz_name, sizeof(mz_name),
 919                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 920                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 921                         errno = ENAMETOOLONG;
 922                         ext_num = 0;
 923                         break;
 924                 }
 925                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 926                                                  socket_id,
 927                                                  RTE_MEMZONE_IOVA_CONTIG |
 928                                                  RTE_MEMZONE_1GB |
 929                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 930                                                  EXTBUF_ZONE_SIZE);
 931                 if (mz == NULL) {
 932                         /*
 933                          * The caller exits on external buffer creation
 934                          * error, so there is no need to free memzones.
 935                          */
 936                         errno = ENOMEM;
 937                         ext_num = 0;
 938                         break;
 939                 }
 940                 xseg->buf_ptr = mz->addr;
 941                 xseg->buf_iova = mz->iova;
 942                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 943                 xseg->elt_size = elt_size;
 944         }
 945         if (ext_num == 0 && xmem != NULL) {
 946                 free(xmem);
 947                 xmem = NULL;
 948         }
 949         *ext_mem = xmem;
 950         return ext_num;
 951 }
 952
 953 /*
 954  * Configuration initialisation done once at init time.
 955  */
 956 static struct rte_mempool *
 957 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 958                  unsigned int socket_id)
 959 {
 960         char pool_name[RTE_MEMPOOL_NAMESIZE];
 961         struct rte_mempool *rte_mp = NULL;
 962         uint32_t mb_size;
 963
 964         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 965         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 966
 967         TESTPMD_LOG(INFO,
 968                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 969                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 970
 971         switch (mp_alloc_type) {
 972         case MP_ALLOC_NATIVE:
 973                 {
 974                         /* wrapper to rte_mempool_create() */
 975                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 976                                         rte_mbuf_best_mempool_ops());
 977                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 978                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 979                         break;
 980                 }
 981         case MP_ALLOC_ANON:
 982                 {
 983                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 984                                 mb_size, (unsigned int) mb_mempool_cache,
 985                                 sizeof(struct rte_pktmbuf_pool_private),
 986                                 socket_id, mempool_flags);
 987                         if (rte_mp == NULL)
 988                                 goto err;
 989
 990                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 991                                 rte_mempool_free(rte_mp);
 992                                 rte_mp = NULL;
 993                                 goto err;
 994                         }
 995                         rte_pktmbuf_pool_init(rte_mp, NULL);
 996                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 997                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 998                         break;
 999                 }
1000         case MP_ALLOC_XMEM:
1001         case MP_ALLOC_XMEM_HUGE:
1002                 {
1003                         int heap_socket;
1004                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1005
1006                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1007                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1008
1009                         heap_socket =
1010                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1011                         if (heap_socket < 0)
1012                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1013
1014                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1015                                         rte_mbuf_best_mempool_ops());
1016                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1017                                         mb_mempool_cache, 0, mbuf_seg_size,
1018                                         heap_socket);
1019                         break;
1020                 }
1021         case MP_ALLOC_XBUF:
1022                 {
1023                         struct rte_pktmbuf_extmem *ext_mem;
1024                         unsigned int ext_num;
1025
1026                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1027                                                socket_id, pool_name, &ext_mem);
1028                         if (ext_num == 0)
1029                                 rte_exit(EXIT_FAILURE,
1030                                          "Can't create pinned data buffers\n");
1031
1032                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1033                                         rte_mbuf_best_mempool_ops());
1034                         rte_mp = rte_pktmbuf_pool_create_extbuf
1035                                         (pool_name, nb_mbuf, mb_mempool_cache,
1036                                          0, mbuf_seg_size, socket_id,
1037                                          ext_mem, ext_num);
1038                         free(ext_mem);
1039                         break;
1040                 }
1041         default:
1042                 {
1043                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1044                 }
1045         }
1046
1047 err:
1048         if (rte_mp == NULL) {
1049                 rte_exit(EXIT_FAILURE,
1050                         "Creation of mbuf pool for socket %u failed: %s\n",
1051                         socket_id, rte_strerror(rte_errno));
1052         } else if (verbose_level > 0) {
1053                 rte_mempool_dump(stdout, rte_mp);
1054         }
1055         return rte_mp;
1056 }
1057
1058 /*
1059  * Check given socket id is valid or not with NUMA mode,
1060  * if valid, return 0, else return -1
1061  */
1062 static int
1063 check_socket_id(const unsigned int socket_id)
1064 {
1065         static int warning_once = 0;
1066
1067         if (new_socket_id(socket_id)) {
1068                 if (!warning_once && numa_support)
1069                         printf("Warning: NUMA should be configured manually by"
1070                                " using --port-numa-config and"
1071                                " --ring-numa-config parameters along with"
1072                                " --numa.\n");
1073                 warning_once = 1;
1074                 return -1;
1075         }
1076         return 0;
1077 }
1078
1079 /*
1080  * Get the allowed maximum number of RX queues.
1081  * *pid return the port id which has minimal value of
1082  * max_rx_queues in all ports.
1083  */
1084 queueid_t
1085 get_allowed_max_nb_rxq(portid_t *pid)
1086 {
1087         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1088         bool max_rxq_valid = false;
1089         portid_t pi;
1090         struct rte_eth_dev_info dev_info;
1091
1092         RTE_ETH_FOREACH_DEV(pi) {
1093                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1094                         continue;
1095
1096                 max_rxq_valid = true;
1097                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1098                         allowed_max_rxq = dev_info.max_rx_queues;
1099                         *pid = pi;
1100                 }
1101         }
1102         return max_rxq_valid ? allowed_max_rxq : 0;
1103 }
1104
1105 /*
1106  * Check input rxq is valid or not.
1107  * If input rxq is not greater than any of maximum number
1108  * of RX queues of all ports, it is valid.
1109  * if valid, return 0, else return -1
1110  */
1111 int
1112 check_nb_rxq(queueid_t rxq)
1113 {
1114         queueid_t allowed_max_rxq;
1115         portid_t pid = 0;
1116
1117         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1118         if (rxq > allowed_max_rxq) {
1119                 printf("Fail: input rxq (%u) can't be greater "
1120                        "than max_rx_queues (%u) of port %u\n",
1121                        rxq,
1122                        allowed_max_rxq,
1123                        pid);
1124                 return -1;
1125         }
1126         return 0;
1127 }
1128
1129 /*
1130  * Get the allowed maximum number of TX queues.
1131  * *pid return the port id which has minimal value of
1132  * max_tx_queues in all ports.
1133  */
1134 queueid_t
1135 get_allowed_max_nb_txq(portid_t *pid)
1136 {
1137         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1138         bool max_txq_valid = false;
1139         portid_t pi;
1140         struct rte_eth_dev_info dev_info;
1141
1142         RTE_ETH_FOREACH_DEV(pi) {
1143                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1144                         continue;
1145
1146                 max_txq_valid = true;
1147                 if (dev_info.max_tx_queues < allowed_max_txq) {
1148                         allowed_max_txq = dev_info.max_tx_queues;
1149                         *pid = pi;
1150                 }
1151         }
1152         return max_txq_valid ? allowed_max_txq : 0;
1153 }
1154
1155 /*
1156  * Check input txq is valid or not.
1157  * If input txq is not greater than any of maximum number
1158  * of TX queues of all ports, it is valid.
1159  * if valid, return 0, else return -1
1160  */
1161 int
1162 check_nb_txq(queueid_t txq)
1163 {
1164         queueid_t allowed_max_txq;
1165         portid_t pid = 0;
1166
1167         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1168         if (txq > allowed_max_txq) {
1169                 printf("Fail: input txq (%u) can't be greater "
1170                        "than max_tx_queues (%u) of port %u\n",
1171                        txq,
1172                        allowed_max_txq,
1173                        pid);
1174                 return -1;
1175         }
1176         return 0;
1177 }
1178
1179 /*
1180  * Get the allowed maximum number of RXDs of every rx queue.
1181  * *pid return the port id which has minimal value of
1182  * max_rxd in all queues of all ports.
1183  */
1184 static uint16_t
1185 get_allowed_max_nb_rxd(portid_t *pid)
1186 {
1187         uint16_t allowed_max_rxd = UINT16_MAX;
1188         portid_t pi;
1189         struct rte_eth_dev_info dev_info;
1190
1191         RTE_ETH_FOREACH_DEV(pi) {
1192                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1193                         continue;
1194
1195                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1196                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1197                         *pid = pi;
1198                 }
1199         }
1200         return allowed_max_rxd;
1201 }
1202
1203 /*
1204  * Get the allowed minimal number of RXDs of every rx queue.
1205  * *pid return the port id which has minimal value of
1206  * min_rxd in all queues of all ports.
1207  */
1208 static uint16_t
1209 get_allowed_min_nb_rxd(portid_t *pid)
1210 {
1211         uint16_t allowed_min_rxd = 0;
1212         portid_t pi;
1213         struct rte_eth_dev_info dev_info;
1214
1215         RTE_ETH_FOREACH_DEV(pi) {
1216                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1217                         continue;
1218
1219                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1220                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1221                         *pid = pi;
1222                 }
1223         }
1224
1225         return allowed_min_rxd;
1226 }
1227
1228 /*
1229  * Check input rxd is valid or not.
1230  * If input rxd is not greater than any of maximum number
1231  * of RXDs of every Rx queues and is not less than any of
1232  * minimal number of RXDs of every Rx queues, it is valid.
1233  * if valid, return 0, else return -1
1234  */
1235 int
1236 check_nb_rxd(queueid_t rxd)
1237 {
1238         uint16_t allowed_max_rxd;
1239         uint16_t allowed_min_rxd;
1240         portid_t pid = 0;
1241
1242         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1243         if (rxd > allowed_max_rxd) {
1244                 printf("Fail: input rxd (%u) can't be greater "
1245                        "than max_rxds (%u) of port %u\n",
1246                        rxd,
1247                        allowed_max_rxd,
1248                        pid);
1249                 return -1;
1250         }
1251
1252         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1253         if (rxd < allowed_min_rxd) {
1254                 printf("Fail: input rxd (%u) can't be less "
1255                        "than min_rxds (%u) of port %u\n",
1256                        rxd,
1257                        allowed_min_rxd,
1258                        pid);
1259                 return -1;
1260         }
1261
1262         return 0;
1263 }
1264
1265 /*
1266  * Get the allowed maximum number of TXDs of every rx queues.
1267  * *pid return the port id which has minimal value of
1268  * max_txd in every tx queue.
1269  */
1270 static uint16_t
1271 get_allowed_max_nb_txd(portid_t *pid)
1272 {
1273         uint16_t allowed_max_txd = UINT16_MAX;
1274         portid_t pi;
1275         struct rte_eth_dev_info dev_info;
1276
1277         RTE_ETH_FOREACH_DEV(pi) {
1278                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1279                         continue;
1280
1281                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1282                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1283                         *pid = pi;
1284                 }
1285         }
1286         return allowed_max_txd;
1287 }
1288
1289 /*
1290  * Get the allowed maximum number of TXDs of every tx queues.
1291  * *pid return the port id which has minimal value of
1292  * min_txd in every tx queue.
1293  */
1294 static uint16_t
1295 get_allowed_min_nb_txd(portid_t *pid)
1296 {
1297         uint16_t allowed_min_txd = 0;
1298         portid_t pi;
1299         struct rte_eth_dev_info dev_info;
1300
1301         RTE_ETH_FOREACH_DEV(pi) {
1302                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1303                         continue;
1304
1305                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1306                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1307                         *pid = pi;
1308                 }
1309         }
1310
1311         return allowed_min_txd;
1312 }
1313
1314 /*
1315  * Check input txd is valid or not.
1316  * If input txd is not greater than any of maximum number
1317  * of TXDs of every Rx queues, it is valid.
1318  * if valid, return 0, else return -1
1319  */
1320 int
1321 check_nb_txd(queueid_t txd)
1322 {
1323         uint16_t allowed_max_txd;
1324         uint16_t allowed_min_txd;
1325         portid_t pid = 0;
1326
1327         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1328         if (txd > allowed_max_txd) {
1329                 printf("Fail: input txd (%u) can't be greater "
1330                        "than max_txds (%u) of port %u\n",
1331                        txd,
1332                        allowed_max_txd,
1333                        pid);
1334                 return -1;
1335         }
1336
1337         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1338         if (txd < allowed_min_txd) {
1339                 printf("Fail: input txd (%u) can't be less "
1340                        "than min_txds (%u) of port %u\n",
1341                        txd,
1342                        allowed_min_txd,
1343                        pid);
1344                 return -1;
1345         }
1346         return 0;
1347 }
1348
1349
1350 /*
1351  * Get the allowed maximum number of hairpin queues.
1352  * *pid return the port id which has minimal value of
1353  * max_hairpin_queues in all ports.
1354  */
1355 queueid_t
1356 get_allowed_max_nb_hairpinq(portid_t *pid)
1357 {
1358         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1359         portid_t pi;
1360         struct rte_eth_hairpin_cap cap;
1361
1362         RTE_ETH_FOREACH_DEV(pi) {
1363                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1364                         *pid = pi;
1365                         return 0;
1366                 }
1367                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1368                         allowed_max_hairpinq = cap.max_nb_queues;
1369                         *pid = pi;
1370                 }
1371         }
1372         return allowed_max_hairpinq;
1373 }
1374
1375 /*
1376  * Check input hairpin is valid or not.
1377  * If input hairpin is not greater than any of maximum number
1378  * of hairpin queues of all ports, it is valid.
1379  * if valid, return 0, else return -1
1380  */
1381 int
1382 check_nb_hairpinq(queueid_t hairpinq)
1383 {
1384         queueid_t allowed_max_hairpinq;
1385         portid_t pid = 0;
1386
1387         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1388         if (hairpinq > allowed_max_hairpinq) {
1389                 printf("Fail: input hairpin (%u) can't be greater "
1390                        "than max_hairpin_queues (%u) of port %u\n",
1391                        hairpinq, allowed_max_hairpinq, pid);
1392                 return -1;
1393         }
1394         return 0;
1395 }
1396
1397 static void
1398 init_config(void)
1399 {
1400         portid_t pid;
1401         struct rte_port *port;
1402         struct rte_mempool *mbp;
1403         unsigned int nb_mbuf_per_pool;
1404         lcoreid_t  lc_id;
1405         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1406         struct rte_gro_param gro_param;
1407         uint32_t gso_types;
1408         uint16_t data_size;
1409         bool warning = 0;
1410         int k;
1411         int ret;
1412
1413         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1414
1415         /* Configuration of logical cores. */
1416         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1417                                 sizeof(struct fwd_lcore *) * nb_lcores,
1418                                 RTE_CACHE_LINE_SIZE);
1419         if (fwd_lcores == NULL) {
1420                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1421                                                         "failed\n", nb_lcores);
1422         }
1423         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1424                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1425                                                sizeof(struct fwd_lcore),
1426                                                RTE_CACHE_LINE_SIZE);
1427                 if (fwd_lcores[lc_id] == NULL) {
1428                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1429                                                                 "failed\n");
1430                 }
1431                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1432         }
1433
1434         RTE_ETH_FOREACH_DEV(pid) {
1435                 port = &ports[pid];
1436                 /* Apply default TxRx configuration for all ports */
1437                 port->dev_conf.txmode = tx_mode;
1438                 port->dev_conf.rxmode = rx_mode;
1439
1440                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1441                 if (ret != 0)
1442                         rte_exit(EXIT_FAILURE,
1443                                  "rte_eth_dev_info_get() failed\n");
1444
1445                 if (!(port->dev_info.tx_offload_capa &
1446                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1447                         port->dev_conf.txmode.offloads &=
1448                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1449                 if (numa_support) {
1450                         if (port_numa[pid] != NUMA_NO_CONFIG)
1451                                 port_per_socket[port_numa[pid]]++;
1452                         else {
1453                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1454
1455                                 /*
1456                                  * if socket_id is invalid,
1457                                  * set to the first available socket.
1458                                  */
1459                                 if (check_socket_id(socket_id) < 0)
1460                                         socket_id = socket_ids[0];
1461                                 port_per_socket[socket_id]++;
1462                         }
1463                 }
1464
1465                 /* Apply Rx offloads configuration */
1466                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1467                         port->rx_conf[k].offloads =
1468                                 port->dev_conf.rxmode.offloads;
1469                 /* Apply Tx offloads configuration */
1470                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1471                         port->tx_conf[k].offloads =
1472                                 port->dev_conf.txmode.offloads;
1473
1474                 /* set flag to initialize port/queue */
1475                 port->need_reconfig = 1;
1476                 port->need_reconfig_queues = 1;
1477                 port->tx_metadata = 0;
1478
1479                 /* Check for maximum number of segments per MTU. Accordingly
1480                  * update the mbuf data size.
1481                  */
1482                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1483                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1484                         data_size = rx_mode.max_rx_pkt_len /
1485                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1486
1487                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1488                                                         mbuf_data_size) {
1489                                 mbuf_data_size = data_size +
1490                                                  RTE_PKTMBUF_HEADROOM;
1491                                 warning = 1;
1492                         }
1493                 }
1494         }
1495
1496         if (warning)
1497                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1498                             mbuf_data_size);
1499
1500         /*
1501          * Create pools of mbuf.
1502          * If NUMA support is disabled, create a single pool of mbuf in
1503          * socket 0 memory by default.
1504          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1505          *
1506          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1507          * nb_txd can be configured at run time.
1508          */
1509         if (param_total_num_mbufs)
1510                 nb_mbuf_per_pool = param_total_num_mbufs;
1511         else {
1512                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1513                         (nb_lcores * mb_mempool_cache) +
1514                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1515                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1516         }
1517
1518         if (numa_support) {
1519                 uint8_t i;
1520
1521                 for (i = 0; i < num_sockets; i++)
1522                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1523                                                        nb_mbuf_per_pool,
1524                                                        socket_ids[i]);
1525         } else {
1526                 if (socket_num == UMA_NO_CONFIG)
1527                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1528                                                        nb_mbuf_per_pool, 0);
1529                 else
1530                         mempools[socket_num] = mbuf_pool_create
1531                                                         (mbuf_data_size,
1532                                                          nb_mbuf_per_pool,
1533                                                          socket_num);
1534         }
1535
1536         init_port_config();
1537
1538         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1539                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1540         /*
1541          * Records which Mbuf pool to use by each logical core, if needed.
1542          */
1543         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1544                 mbp = mbuf_pool_find(
1545                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1546
1547                 if (mbp == NULL)
1548                         mbp = mbuf_pool_find(0);
1549                 fwd_lcores[lc_id]->mbp = mbp;
1550                 /* initialize GSO context */
1551                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1552                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1553                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1554                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1555                         RTE_ETHER_CRC_LEN;
1556                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1557         }
1558
1559         /* Configuration of packet forwarding streams. */
1560         if (init_fwd_streams() < 0)
1561                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1562
1563         fwd_config_setup();
1564
1565         /* create a gro context for each lcore */
1566         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1567         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1568         gro_param.max_item_per_flow = MAX_PKT_BURST;
1569         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1570                 gro_param.socket_id = rte_lcore_to_socket_id(
1571                                 fwd_lcores_cpuids[lc_id]);
1572                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1573                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1574                         rte_exit(EXIT_FAILURE,
1575                                         "rte_gro_ctx_create() failed\n");
1576                 }
1577         }
1578 }
1579
1580
1581 void
1582 reconfig(portid_t new_port_id, unsigned socket_id)
1583 {
1584         struct rte_port *port;
1585         int ret;
1586
1587         /* Reconfiguration of Ethernet ports. */
1588         port = &ports[new_port_id];
1589
1590         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1591         if (ret != 0)
1592                 return;
1593
1594         /* set flag to initialize port/queue */
1595         port->need_reconfig = 1;
1596         port->need_reconfig_queues = 1;
1597         port->socket_id = socket_id;
1598
1599         init_port_config();
1600 }
1601
1602
1603 int
1604 init_fwd_streams(void)
1605 {
1606         portid_t pid;
1607         struct rte_port *port;
1608         streamid_t sm_id, nb_fwd_streams_new;
1609         queueid_t q;
1610
1611         /* set socket id according to numa or not */
1612         RTE_ETH_FOREACH_DEV(pid) {
1613                 port = &ports[pid];
1614                 if (nb_rxq > port->dev_info.max_rx_queues) {
1615                         printf("Fail: nb_rxq(%d) is greater than "
1616                                 "max_rx_queues(%d)\n", nb_rxq,
1617                                 port->dev_info.max_rx_queues);
1618                         return -1;
1619                 }
1620                 if (nb_txq > port->dev_info.max_tx_queues) {
1621                         printf("Fail: nb_txq(%d) is greater than "
1622                                 "max_tx_queues(%d)\n", nb_txq,
1623                                 port->dev_info.max_tx_queues);
1624                         return -1;
1625                 }
1626                 if (numa_support) {
1627                         if (port_numa[pid] != NUMA_NO_CONFIG)
1628                                 port->socket_id = port_numa[pid];
1629                         else {
1630                                 port->socket_id = rte_eth_dev_socket_id(pid);
1631
1632                                 /*
1633                                  * if socket_id is invalid,
1634                                  * set to the first available socket.
1635                                  */
1636                                 if (check_socket_id(port->socket_id) < 0)
1637                                         port->socket_id = socket_ids[0];
1638                         }
1639                 }
1640                 else {
1641                         if (socket_num == UMA_NO_CONFIG)
1642                                 port->socket_id = 0;
1643                         else
1644                                 port->socket_id = socket_num;
1645                 }
1646         }
1647
1648         q = RTE_MAX(nb_rxq, nb_txq);
1649         if (q == 0) {
1650                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1651                 return -1;
1652         }
1653         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1654         if (nb_fwd_streams_new == nb_fwd_streams)
1655                 return 0;
1656         /* clear the old */
1657         if (fwd_streams != NULL) {
1658                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1659                         if (fwd_streams[sm_id] == NULL)
1660                                 continue;
1661                         rte_free(fwd_streams[sm_id]);
1662                         fwd_streams[sm_id] = NULL;
1663                 }
1664                 rte_free(fwd_streams);
1665                 fwd_streams = NULL;
1666         }
1667
1668         /* init new */
1669         nb_fwd_streams = nb_fwd_streams_new;
1670         if (nb_fwd_streams) {
1671                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1672                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1673                         RTE_CACHE_LINE_SIZE);
1674                 if (fwd_streams == NULL)
1675                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1676                                  " (struct fwd_stream *)) failed\n",
1677                                  nb_fwd_streams);
1678
1679                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1680                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1681                                 " struct fwd_stream", sizeof(struct fwd_stream),
1682                                 RTE_CACHE_LINE_SIZE);
1683                         if (fwd_streams[sm_id] == NULL)
1684                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1685                                          "(struct fwd_stream) failed\n");
1686                 }
1687         }
1688
1689         return 0;
1690 }
1691
1692 static void
1693 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1694 {
1695         uint64_t total_burst, sburst;
1696         uint64_t nb_burst;
1697         uint64_t burst_stats[4];
1698         uint16_t pktnb_stats[4];
1699         uint16_t nb_pkt;
1700         int burst_percent[4], sburstp;
1701         int i;
1702
1703         /*
1704          * First compute the total number of packet bursts and the
1705          * two highest numbers of bursts of the same number of packets.
1706          */
1707         memset(&burst_stats, 0x0, sizeof(burst_stats));
1708         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1709
1710         /* Show stats for 0 burst size always */
1711         total_burst = pbs->pkt_burst_spread[0];
1712         burst_stats[0] = pbs->pkt_burst_spread[0];
1713         pktnb_stats[0] = 0;
1714
1715         /* Find the next 2 burst sizes with highest occurrences. */
1716         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1717                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1718
1719                 if (nb_burst == 0)
1720                         continue;
1721
1722                 total_burst += nb_burst;
1723
1724                 if (nb_burst > burst_stats[1]) {
1725                         burst_stats[2] = burst_stats[1];
1726                         pktnb_stats[2] = pktnb_stats[1];
1727                         burst_stats[1] = nb_burst;
1728                         pktnb_stats[1] = nb_pkt;
1729                 } else if (nb_burst > burst_stats[2]) {
1730                         burst_stats[2] = nb_burst;
1731                         pktnb_stats[2] = nb_pkt;
1732                 }
1733         }
1734         if (total_burst == 0)
1735                 return;
1736
1737         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1738         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1739                 if (i == 3) {
1740                         printf("%d%% of other]\n", 100 - sburstp);
1741                         return;
1742                 }
1743
1744                 sburst += burst_stats[i];
1745                 if (sburst == total_burst) {
1746                         printf("%d%% of %d pkts]\n",
1747                                 100 - sburstp, (int) pktnb_stats[i]);
1748                         return;
1749                 }
1750
1751                 burst_percent[i] =
1752                         (double)burst_stats[i] / total_burst * 100;
1753                 printf("%d%% of %d pkts + ",
1754                         burst_percent[i], (int) pktnb_stats[i]);
1755                 sburstp += burst_percent[i];
1756         }
1757 }
1758
1759 static void
1760 fwd_stream_stats_display(streamid_t stream_id)
1761 {
1762         struct fwd_stream *fs;
1763         static const char *fwd_top_stats_border = "-------";
1764
1765         fs = fwd_streams[stream_id];
1766         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1767             (fs->fwd_dropped == 0))
1768                 return;
1769         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1770                "TX Port=%2d/Queue=%2d %s\n",
1771                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1772                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1773         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1774                " TX-dropped: %-14"PRIu64,
1775                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1776
1777         /* if checksum mode */
1778         if (cur_fwd_eng == &csum_fwd_engine) {
1779                 printf("  RX- bad IP checksum: %-14"PRIu64
1780                        "  Rx- bad L4 checksum: %-14"PRIu64
1781                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1782                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1783                         fs->rx_bad_outer_l4_csum);
1784         } else {
1785                 printf("\n");
1786         }
1787
1788         if (record_burst_stats) {
1789                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1790                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1791         }
1792 }
1793
1794 void
1795 fwd_stats_display(void)
1796 {
1797         static const char *fwd_stats_border = "----------------------";
1798         static const char *acc_stats_border = "+++++++++++++++";
1799         struct {
1800                 struct fwd_stream *rx_stream;
1801                 struct fwd_stream *tx_stream;
1802                 uint64_t tx_dropped;
1803                 uint64_t rx_bad_ip_csum;
1804                 uint64_t rx_bad_l4_csum;
1805                 uint64_t rx_bad_outer_l4_csum;
1806         } ports_stats[RTE_MAX_ETHPORTS];
1807         uint64_t total_rx_dropped = 0;
1808         uint64_t total_tx_dropped = 0;
1809         uint64_t total_rx_nombuf = 0;
1810         struct rte_eth_stats stats;
1811         uint64_t fwd_cycles = 0;
1812         uint64_t total_recv = 0;
1813         uint64_t total_xmit = 0;
1814         struct rte_port *port;
1815         streamid_t sm_id;
1816         portid_t pt_id;
1817         int i;
1818
1819         memset(ports_stats, 0, sizeof(ports_stats));
1820
1821         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1822                 struct fwd_stream *fs = fwd_streams[sm_id];
1823
1824                 if (cur_fwd_config.nb_fwd_streams >
1825                     cur_fwd_config.nb_fwd_ports) {
1826                         fwd_stream_stats_display(sm_id);
1827                 } else {
1828                         ports_stats[fs->tx_port].tx_stream = fs;
1829                         ports_stats[fs->rx_port].rx_stream = fs;
1830                 }
1831
1832                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1833
1834                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1835                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1836                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1837                                 fs->rx_bad_outer_l4_csum;
1838
1839                 if (record_core_cycles)
1840                         fwd_cycles += fs->core_cycles;
1841         }
1842         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1843                 uint8_t j;
1844
1845                 pt_id = fwd_ports_ids[i];
1846                 port = &ports[pt_id];
1847
1848                 rte_eth_stats_get(pt_id, &stats);
1849                 stats.ipackets -= port->stats.ipackets;
1850                 stats.opackets -= port->stats.opackets;
1851                 stats.ibytes -= port->stats.ibytes;
1852                 stats.obytes -= port->stats.obytes;
1853                 stats.imissed -= port->stats.imissed;
1854                 stats.oerrors -= port->stats.oerrors;
1855                 stats.rx_nombuf -= port->stats.rx_nombuf;
1856
1857                 total_recv += stats.ipackets;
1858                 total_xmit += stats.opackets;
1859                 total_rx_dropped += stats.imissed;
1860                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1861                 total_tx_dropped += stats.oerrors;
1862                 total_rx_nombuf  += stats.rx_nombuf;
1863
1864                 printf("\n  %s Forward statistics for port %-2d %s\n",
1865                        fwd_stats_border, pt_id, fwd_stats_border);
1866
1867                 if (!port->rx_queue_stats_mapping_enabled &&
1868                     !port->tx_queue_stats_mapping_enabled) {
1869                         printf("  RX-packets: %-14"PRIu64
1870                                " RX-dropped: %-14"PRIu64
1871                                "RX-total: %-"PRIu64"\n",
1872                                stats.ipackets, stats.imissed,
1873                                stats.ipackets + stats.imissed);
1874
1875                         if (cur_fwd_eng == &csum_fwd_engine)
1876                                 printf("  Bad-ipcsum: %-14"PRIu64
1877                                        " Bad-l4csum: %-14"PRIu64
1878                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1879                                        ports_stats[pt_id].rx_bad_ip_csum,
1880                                        ports_stats[pt_id].rx_bad_l4_csum,
1881                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1882                         if (stats.ierrors + stats.rx_nombuf > 0) {
1883                                 printf("  RX-error: %-"PRIu64"\n",
1884                                        stats.ierrors);
1885                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1886                                        stats.rx_nombuf);
1887                         }
1888
1889                         printf("  TX-packets: %-14"PRIu64
1890                                " TX-dropped: %-14"PRIu64
1891                                "TX-total: %-"PRIu64"\n",
1892                                stats.opackets, ports_stats[pt_id].tx_dropped,
1893                                stats.opackets + ports_stats[pt_id].tx_dropped);
1894                 } else {
1895                         printf("  RX-packets:             %14"PRIu64
1896                                "    RX-dropped:%14"PRIu64
1897                                "    RX-total:%14"PRIu64"\n",
1898                                stats.ipackets, stats.imissed,
1899                                stats.ipackets + stats.imissed);
1900
1901                         if (cur_fwd_eng == &csum_fwd_engine)
1902                                 printf("  Bad-ipcsum:%14"PRIu64
1903                                        "    Bad-l4csum:%14"PRIu64
1904                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1905                                        ports_stats[pt_id].rx_bad_ip_csum,
1906                                        ports_stats[pt_id].rx_bad_l4_csum,
1907                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1908                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1909                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1910                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1911                                        stats.rx_nombuf);
1912                         }
1913
1914                         printf("  TX-packets:             %14"PRIu64
1915                                "    TX-dropped:%14"PRIu64
1916                                "    TX-total:%14"PRIu64"\n",
1917                                stats.opackets, ports_stats[pt_id].tx_dropped,
1918                                stats.opackets + ports_stats[pt_id].tx_dropped);
1919                 }
1920
1921                 if (record_burst_stats) {
1922                         if (ports_stats[pt_id].rx_stream)
1923                                 pkt_burst_stats_display("RX",
1924                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1925                         if (ports_stats[pt_id].tx_stream)
1926                                 pkt_burst_stats_display("TX",
1927                                         &ports_stats[pt_id].tx_stream->tx_burst_stats);
1928                 }
1929
1930                 if (port->rx_queue_stats_mapping_enabled) {
1931                         printf("\n");
1932                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1933                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1934                                        "     RX-errors:%14"PRIu64
1935                                        "    RX-bytes:%14"PRIu64"\n",
1936                                        j, stats.q_ipackets[j],
1937                                        stats.q_errors[j], stats.q_ibytes[j]);
1938                         }
1939                         printf("\n");
1940                 }
1941                 if (port->tx_queue_stats_mapping_enabled) {
1942                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1943                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1944                                        "                                 TX-bytes:%14"
1945                                        PRIu64"\n",
1946                                        j, stats.q_opackets[j],
1947                                        stats.q_obytes[j]);
1948                         }
1949                 }
1950
1951                 printf("  %s--------------------------------%s\n",
1952                        fwd_stats_border, fwd_stats_border);
1953         }
1954
1955         printf("\n  %s Accumulated forward statistics for all ports"
1956                "%s\n",
1957                acc_stats_border, acc_stats_border);
1958         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1959                "%-"PRIu64"\n"
1960                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1961                "%-"PRIu64"\n",
1962                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1963                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1964         if (total_rx_nombuf > 0)
1965                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1966         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1967                "%s\n",
1968                acc_stats_border, acc_stats_border);
1969         if (record_core_cycles) {
1970 #define CYC_PER_MHZ 1E6
1971                 if (total_recv > 0 || total_xmit > 0) {
1972                         uint64_t total_pkts = 0;
1973                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1974                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1975                                 total_pkts = total_xmit;
1976                         else
1977                                 total_pkts = total_recv;
1978
1979                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1980                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1981                                " MHz Clock\n",
1982                                (double) fwd_cycles / total_pkts,
1983                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1984                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1985                 }
1986         }
1987 }
1988
1989 void
1990 fwd_stats_reset(void)
1991 {
1992         streamid_t sm_id;
1993         portid_t pt_id;
1994         int i;
1995
1996         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1997                 pt_id = fwd_ports_ids[i];
1998                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1999         }
2000         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2001                 struct fwd_stream *fs = fwd_streams[sm_id];
2002
2003                 fs->rx_packets = 0;
2004                 fs->tx_packets = 0;
2005                 fs->fwd_dropped = 0;
2006                 fs->rx_bad_ip_csum = 0;
2007                 fs->rx_bad_l4_csum = 0;
2008                 fs->rx_bad_outer_l4_csum = 0;
2009
2010                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2011                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2012                 fs->core_cycles = 0;
2013         }
2014 }
2015
2016 static void
2017 flush_fwd_rx_queues(void)
2018 {
2019         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2020         portid_t  rxp;
2021         portid_t port_id;
2022         queueid_t rxq;
2023         uint16_t  nb_rx;
2024         uint16_t  i;
2025         uint8_t   j;
2026         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2027         uint64_t timer_period;
2028
2029         /* convert to number of cycles */
2030         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2031
2032         for (j = 0; j < 2; j++) {
2033                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2034                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2035                                 port_id = fwd_ports_ids[rxp];
2036                                 /**
2037                                 * testpmd can stuck in the below do while loop
2038                                 * if rte_eth_rx_burst() always returns nonzero
2039                                 * packets. So timer is added to exit this loop
2040                                 * after 1sec timer expiry.
2041                                 */
2042                                 prev_tsc = rte_rdtsc();
2043                                 do {
2044                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2045                                                 pkts_burst, MAX_PKT_BURST);
2046                                         for (i = 0; i < nb_rx; i++)
2047                                                 rte_pktmbuf_free(pkts_burst[i]);
2048
2049                                         cur_tsc = rte_rdtsc();
2050                                         diff_tsc = cur_tsc - prev_tsc;
2051                                         timer_tsc += diff_tsc;
2052                                 } while ((nb_rx > 0) &&
2053                                         (timer_tsc < timer_period));
2054                                 timer_tsc = 0;
2055                         }
2056                 }
2057                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2058         }
2059 }
2060
2061 static void
2062 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2063 {
2064         struct fwd_stream **fsm;
2065         streamid_t nb_fs;
2066         streamid_t sm_id;
2067 #ifdef RTE_LIBRTE_BITRATESTATS
2068         uint64_t tics_per_1sec;
2069         uint64_t tics_datum;
2070         uint64_t tics_current;
2071         uint16_t i, cnt_ports;
2072
2073         cnt_ports = nb_ports;
2074         tics_datum = rte_rdtsc();
2075         tics_per_1sec = rte_get_timer_hz();
2076 #endif
2077         fsm = &fwd_streams[fc->stream_idx];
2078         nb_fs = fc->stream_nb;
2079         do {
2080                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2081                         (*pkt_fwd)(fsm[sm_id]);
2082 #ifdef RTE_LIBRTE_BITRATESTATS
2083                 if (bitrate_enabled != 0 &&
2084                                 bitrate_lcore_id == rte_lcore_id()) {
2085                         tics_current = rte_rdtsc();
2086                         if (tics_current - tics_datum >= tics_per_1sec) {
2087                                 /* Periodic bitrate calculation */
2088                                 for (i = 0; i < cnt_ports; i++)
2089                                         rte_stats_bitrate_calc(bitrate_data,
2090                                                 ports_ids[i]);
2091                                 tics_datum = tics_current;
2092                         }
2093                 }
2094 #endif
2095 #ifdef RTE_LIBRTE_LATENCY_STATS
2096                 if (latencystats_enabled != 0 &&
2097                                 latencystats_lcore_id == rte_lcore_id())
2098                         rte_latencystats_update();
2099 #endif
2100
2101         } while (! fc->stopped);
2102 }
2103
2104 static int
2105 start_pkt_forward_on_core(void *fwd_arg)
2106 {
2107         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2108                              cur_fwd_config.fwd_eng->packet_fwd);
2109         return 0;
2110 }
2111
2112 /*
2113  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2114  * Used to start communication flows in network loopback test configurations.
2115  */
2116 static int
2117 run_one_txonly_burst_on_core(void *fwd_arg)
2118 {
2119         struct fwd_lcore *fwd_lc;
2120         struct fwd_lcore tmp_lcore;
2121
2122         fwd_lc = (struct fwd_lcore *) fwd_arg;
2123         tmp_lcore = *fwd_lc;
2124         tmp_lcore.stopped = 1;
2125         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2126         return 0;
2127 }
2128
2129 /*
2130  * Launch packet forwarding:
2131  *     - Setup per-port forwarding context.
2132  *     - launch logical cores with their forwarding configuration.
2133  */
2134 static void
2135 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2136 {
2137         port_fwd_begin_t port_fwd_begin;
2138         unsigned int i;
2139         unsigned int lc_id;
2140         int diag;
2141
2142         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2143         if (port_fwd_begin != NULL) {
2144                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2145                         (*port_fwd_begin)(fwd_ports_ids[i]);
2146         }
2147         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2148                 lc_id = fwd_lcores_cpuids[i];
2149                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2150                         fwd_lcores[i]->stopped = 0;
2151                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2152                                                      fwd_lcores[i], lc_id);
2153                         if (diag != 0)
2154                                 printf("launch lcore %u failed - diag=%d\n",
2155                                        lc_id, diag);
2156                 }
2157         }
2158 }
2159
2160 /*
2161  * Launch packet forwarding configuration.
2162  */
2163 void
2164 start_packet_forwarding(int with_tx_first)
2165 {
2166         port_fwd_begin_t port_fwd_begin;
2167         port_fwd_end_t  port_fwd_end;
2168         struct rte_port *port;
2169         unsigned int i;
2170         portid_t   pt_id;
2171
2172         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2173                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2174
2175         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2176                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2177
2178         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2179                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2180                 (!nb_rxq || !nb_txq))
2181                 rte_exit(EXIT_FAILURE,
2182                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2183                         cur_fwd_eng->fwd_mode_name);
2184
2185         if (all_ports_started() == 0) {
2186                 printf("Not all ports were started\n");
2187                 return;
2188         }
2189         if (test_done == 0) {
2190                 printf("Packet forwarding already started\n");
2191                 return;
2192         }
2193
2194
2195         if(dcb_test) {
2196                 for (i = 0; i < nb_fwd_ports; i++) {
2197                         pt_id = fwd_ports_ids[i];
2198                         port = &ports[pt_id];
2199                         if (!port->dcb_flag) {
2200                                 printf("In DCB mode, all forwarding ports must "
2201                                        "be configured in this mode.\n");
2202                                 return;
2203                         }
2204                 }
2205                 if (nb_fwd_lcores == 1) {
2206                         printf("In DCB mode,the nb forwarding cores "
2207                                "should be larger than 1.\n");
2208                         return;
2209                 }
2210         }
2211         test_done = 0;
2212
2213         fwd_config_setup();
2214
2215         if(!no_flush_rx)
2216                 flush_fwd_rx_queues();
2217
2218         pkt_fwd_config_display(&cur_fwd_config);
2219         rxtx_config_display();
2220
2221         fwd_stats_reset();
2222         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2223                 pt_id = fwd_ports_ids[i];
2224                 port = &ports[pt_id];
2225                 map_port_queue_stats_mapping_registers(pt_id, port);
2226         }
2227         if (with_tx_first) {
2228                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2229                 if (port_fwd_begin != NULL) {
2230                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2231                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2232                 }
2233                 while (with_tx_first--) {
2234                         launch_packet_forwarding(
2235                                         run_one_txonly_burst_on_core);
2236                         rte_eal_mp_wait_lcore();
2237                 }
2238                 port_fwd_end = tx_only_engine.port_fwd_end;
2239                 if (port_fwd_end != NULL) {
2240                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2241                                 (*port_fwd_end)(fwd_ports_ids[i]);
2242                 }
2243         }
2244         launch_packet_forwarding(start_pkt_forward_on_core);
2245 }
2246
2247 void
2248 stop_packet_forwarding(void)
2249 {
2250         port_fwd_end_t port_fwd_end;
2251         lcoreid_t lc_id;
2252         portid_t pt_id;
2253         int i;
2254
2255         if (test_done) {
2256                 printf("Packet forwarding not started\n");
2257                 return;
2258         }
2259         printf("Telling cores to stop...");
2260         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2261                 fwd_lcores[lc_id]->stopped = 1;
2262         printf("\nWaiting for lcores to finish...\n");
2263         rte_eal_mp_wait_lcore();
2264         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2265         if (port_fwd_end != NULL) {
2266                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2267                         pt_id = fwd_ports_ids[i];
2268                         (*port_fwd_end)(pt_id);
2269                 }
2270         }
2271
2272         fwd_stats_display();
2273
2274         printf("\nDone.\n");
2275         test_done = 1;
2276 }
2277
2278 void
2279 dev_set_link_up(portid_t pid)
2280 {
2281         if (rte_eth_dev_set_link_up(pid) < 0)
2282                 printf("\nSet link up fail.\n");
2283 }
2284
2285 void
2286 dev_set_link_down(portid_t pid)
2287 {
2288         if (rte_eth_dev_set_link_down(pid) < 0)
2289                 printf("\nSet link down fail.\n");
2290 }
2291
2292 static int
2293 all_ports_started(void)
2294 {
2295         portid_t pi;
2296         struct rte_port *port;
2297
2298         RTE_ETH_FOREACH_DEV(pi) {
2299                 port = &ports[pi];
2300                 /* Check if there is a port which is not started */
2301                 if ((port->port_status != RTE_PORT_STARTED) &&
2302                         (port->slave_flag == 0))
2303                         return 0;
2304         }
2305
2306         /* No port is not started */
2307         return 1;
2308 }
2309
2310 int
2311 port_is_stopped(portid_t port_id)
2312 {
2313         struct rte_port *port = &ports[port_id];
2314
2315         if ((port->port_status != RTE_PORT_STOPPED) &&
2316             (port->slave_flag == 0))
2317                 return 0;
2318         return 1;
2319 }
2320
2321 int
2322 all_ports_stopped(void)
2323 {
2324         portid_t pi;
2325
2326         RTE_ETH_FOREACH_DEV(pi) {
2327                 if (!port_is_stopped(pi))
2328                         return 0;
2329         }
2330
2331         return 1;
2332 }
2333
2334 int
2335 port_is_started(portid_t port_id)
2336 {
2337         if (port_id_is_invalid(port_id, ENABLED_WARN))
2338                 return 0;
2339
2340         if (ports[port_id].port_status != RTE_PORT_STARTED)
2341                 return 0;
2342
2343         return 1;
2344 }
2345
2346 /* Configure the Rx and Tx hairpin queues for the selected port. */
2347 static int
2348 setup_hairpin_queues(portid_t pi)
2349 {
2350         queueid_t qi;
2351         struct rte_eth_hairpin_conf hairpin_conf = {
2352                 .peer_count = 1,
2353         };
2354         int i;
2355         int diag;
2356         struct rte_port *port = &ports[pi];
2357
2358         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2359                 hairpin_conf.peers[0].port = pi;
2360                 hairpin_conf.peers[0].queue = i + nb_rxq;
2361                 diag = rte_eth_tx_hairpin_queue_setup
2362                         (pi, qi, nb_txd, &hairpin_conf);
2363                 i++;
2364                 if (diag == 0)
2365                         continue;
2366
2367                 /* Fail to setup rx queue, return */
2368                 if (rte_atomic16_cmpset(&(port->port_status),
2369                                         RTE_PORT_HANDLING,
2370                                         RTE_PORT_STOPPED) == 0)
2371                         printf("Port %d can not be set back "
2372                                         "to stopped\n", pi);
2373                 printf("Fail to configure port %d hairpin "
2374                                 "queues\n", pi);
2375                 /* try to reconfigure queues next time */
2376                 port->need_reconfig_queues = 1;
2377                 return -1;
2378         }
2379         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2380                 hairpin_conf.peers[0].port = pi;
2381                 hairpin_conf.peers[0].queue = i + nb_txq;
2382                 diag = rte_eth_rx_hairpin_queue_setup
2383                         (pi, qi, nb_rxd, &hairpin_conf);
2384                 i++;
2385                 if (diag == 0)
2386                         continue;
2387
2388                 /* Fail to setup rx queue, return */
2389                 if (rte_atomic16_cmpset(&(port->port_status),
2390                                         RTE_PORT_HANDLING,
2391                                         RTE_PORT_STOPPED) == 0)
2392                         printf("Port %d can not be set back "
2393                                         "to stopped\n", pi);
2394                 printf("Fail to configure port %d hairpin "
2395                                 "queues\n", pi);
2396                 /* try to reconfigure queues next time */
2397                 port->need_reconfig_queues = 1;
2398                 return -1;
2399         }
2400         return 0;
2401 }
2402
2403 int
2404 start_port(portid_t pid)
2405 {
2406         int diag, need_check_link_status = -1;
2407         portid_t pi;
2408         queueid_t qi;
2409         struct rte_port *port;
2410         struct rte_ether_addr mac_addr;
2411         struct rte_eth_hairpin_cap cap;
2412
2413         if (port_id_is_invalid(pid, ENABLED_WARN))
2414                 return 0;
2415
2416         if(dcb_config)
2417                 dcb_test = 1;
2418         RTE_ETH_FOREACH_DEV(pi) {
2419                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2420                         continue;
2421
2422                 need_check_link_status = 0;
2423                 port = &ports[pi];
2424                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2425                                                  RTE_PORT_HANDLING) == 0) {
2426                         printf("Port %d is now not stopped\n", pi);
2427                         continue;
2428                 }
2429
2430                 if (port->need_reconfig > 0) {
2431                         port->need_reconfig = 0;
2432
2433                         if (flow_isolate_all) {
2434                                 int ret = port_flow_isolate(pi, 1);
2435                                 if (ret) {
2436                                         printf("Failed to apply isolated"
2437                                                " mode on port %d\n", pi);
2438                                         return -1;
2439                                 }
2440                         }
2441                         configure_rxtx_dump_callbacks(0);
2442                         printf("Configuring Port %d (socket %u)\n", pi,
2443                                         port->socket_id);
2444                         if (nb_hairpinq > 0 &&
2445                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2446                                 printf("Port %d doesn't support hairpin "
2447                                        "queues\n", pi);
2448                                 return -1;
2449                         }
2450                         /* configure port */
2451                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2452                                                      nb_txq + nb_hairpinq,
2453                                                      &(port->dev_conf));
2454                         if (diag != 0) {
2455                                 if (rte_atomic16_cmpset(&(port->port_status),
2456                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2457                                         printf("Port %d can not be set back "
2458                                                         "to stopped\n", pi);
2459                                 printf("Fail to configure port %d\n", pi);
2460                                 /* try to reconfigure port next time */
2461                                 port->need_reconfig = 1;
2462                                 return -1;
2463                         }
2464                 }
2465                 if (port->need_reconfig_queues > 0) {
2466                         port->need_reconfig_queues = 0;
2467                         /* setup tx queues */
2468                         for (qi = 0; qi < nb_txq; qi++) {
2469                                 if ((numa_support) &&
2470                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2471                                         diag = rte_eth_tx_queue_setup(pi, qi,
2472                                                 port->nb_tx_desc[qi],
2473                                                 txring_numa[pi],
2474                                                 &(port->tx_conf[qi]));
2475                                 else
2476                                         diag = rte_eth_tx_queue_setup(pi, qi,
2477                                                 port->nb_tx_desc[qi],
2478                                                 port->socket_id,
2479                                                 &(port->tx_conf[qi]));
2480
2481                                 if (diag == 0)
2482                                         continue;
2483
2484                                 /* Fail to setup tx queue, return */
2485                                 if (rte_atomic16_cmpset(&(port->port_status),
2486                                                         RTE_PORT_HANDLING,
2487                                                         RTE_PORT_STOPPED) == 0)
2488                                         printf("Port %d can not be set back "
2489                                                         "to stopped\n", pi);
2490                                 printf("Fail to configure port %d tx queues\n",
2491                                        pi);
2492                                 /* try to reconfigure queues next time */
2493                                 port->need_reconfig_queues = 1;
2494                                 return -1;
2495                         }
2496                         for (qi = 0; qi < nb_rxq; qi++) {
2497                                 /* setup rx queues */
2498                                 if ((numa_support) &&
2499                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2500                                         struct rte_mempool * mp =
2501                                                 mbuf_pool_find(rxring_numa[pi]);
2502                                         if (mp == NULL) {
2503                                                 printf("Failed to setup RX queue:"
2504                                                         "No mempool allocation"
2505                                                         " on the socket %d\n",
2506                                                         rxring_numa[pi]);
2507                                                 return -1;
2508                                         }
2509
2510                                         diag = rte_eth_rx_queue_setup(pi, qi,
2511                                              port->nb_rx_desc[qi],
2512                                              rxring_numa[pi],
2513                                              &(port->rx_conf[qi]),
2514                                              mp);
2515                                 } else {
2516                                         struct rte_mempool *mp =
2517                                                 mbuf_pool_find(port->socket_id);
2518                                         if (mp == NULL) {
2519                                                 printf("Failed to setup RX queue:"
2520                                                         "No mempool allocation"
2521                                                         " on the socket %d\n",
2522                                                         port->socket_id);
2523                                                 return -1;
2524                                         }
2525                                         diag = rte_eth_rx_queue_setup(pi, qi,
2526                                              port->nb_rx_desc[qi],
2527                                              port->socket_id,
2528                                              &(port->rx_conf[qi]),
2529                                              mp);
2530                                 }
2531                                 if (diag == 0)
2532                                         continue;
2533
2534                                 /* Fail to setup rx queue, return */
2535                                 if (rte_atomic16_cmpset(&(port->port_status),
2536                                                         RTE_PORT_HANDLING,
2537                                                         RTE_PORT_STOPPED) == 0)
2538                                         printf("Port %d can not be set back "
2539                                                         "to stopped\n", pi);
2540                                 printf("Fail to configure port %d rx queues\n",
2541                                        pi);
2542                                 /* try to reconfigure queues next time */
2543                                 port->need_reconfig_queues = 1;
2544                                 return -1;
2545                         }
2546                         /* setup hairpin queues */
2547                         if (setup_hairpin_queues(pi) != 0)
2548                                 return -1;
2549                 }
2550                 configure_rxtx_dump_callbacks(verbose_level);
2551                 if (clear_ptypes) {
2552                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2553                                         NULL, 0);
2554                         if (diag < 0)
2555                                 printf(
2556                                 "Port %d: Failed to disable Ptype parsing\n",
2557                                 pi);
2558                 }
2559
2560                 /* start port */
2561                 if (rte_eth_dev_start(pi) < 0) {
2562                         printf("Fail to start port %d\n", pi);
2563
2564                         /* Fail to setup rx queue, return */
2565                         if (rte_atomic16_cmpset(&(port->port_status),
2566                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2567                                 printf("Port %d can not be set back to "
2568                                                         "stopped\n", pi);
2569                         continue;
2570                 }
2571
2572                 if (rte_atomic16_cmpset(&(port->port_status),
2573                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2574                         printf("Port %d can not be set into started\n", pi);
2575
2576                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2577                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2578                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2579                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2580                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2581
2582                 /* at least one port started, need checking link status */
2583                 need_check_link_status = 1;
2584         }
2585
2586         if (need_check_link_status == 1 && !no_link_check)
2587                 check_all_ports_link_status(RTE_PORT_ALL);
2588         else if (need_check_link_status == 0)
2589                 printf("Please stop the ports first\n");
2590
2591         printf("Done\n");
2592         return 0;
2593 }
2594
2595 void
2596 stop_port(portid_t pid)
2597 {
2598         portid_t pi;
2599         struct rte_port *port;
2600         int need_check_link_status = 0;
2601
2602         if (dcb_test) {
2603                 dcb_test = 0;
2604                 dcb_config = 0;
2605         }
2606
2607         if (port_id_is_invalid(pid, ENABLED_WARN))
2608                 return;
2609
2610         printf("Stopping ports...\n");
2611
2612         RTE_ETH_FOREACH_DEV(pi) {
2613                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2614                         continue;
2615
2616                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2617                         printf("Please remove port %d from forwarding configuration.\n", pi);
2618                         continue;
2619                 }
2620
2621                 if (port_is_bonding_slave(pi)) {
2622                         printf("Please remove port %d from bonded device.\n", pi);
2623                         continue;
2624                 }
2625
2626                 port = &ports[pi];
2627                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2628                                                 RTE_PORT_HANDLING) == 0)
2629                         continue;
2630
2631                 rte_eth_dev_stop(pi);
2632
2633                 if (rte_atomic16_cmpset(&(port->port_status),
2634                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2635                         printf("Port %d can not be set into stopped\n", pi);
2636                 need_check_link_status = 1;
2637         }
2638         if (need_check_link_status && !no_link_check)
2639                 check_all_ports_link_status(RTE_PORT_ALL);
2640
2641         printf("Done\n");
2642 }
2643
2644 static void
2645 remove_invalid_ports_in(portid_t *array, portid_t *total)
2646 {
2647         portid_t i;
2648         portid_t new_total = 0;
2649
2650         for (i = 0; i < *total; i++)
2651                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2652                         array[new_total] = array[i];
2653                         new_total++;
2654                 }
2655         *total = new_total;
2656 }
2657
2658 static void
2659 remove_invalid_ports(void)
2660 {
2661         remove_invalid_ports_in(ports_ids, &nb_ports);
2662         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2663         nb_cfg_ports = nb_fwd_ports;
2664 }
2665
2666 void
2667 close_port(portid_t pid)
2668 {
2669         portid_t pi;
2670         struct rte_port *port;
2671
2672         if (port_id_is_invalid(pid, ENABLED_WARN))
2673                 return;
2674
2675         printf("Closing ports...\n");
2676
2677         RTE_ETH_FOREACH_DEV(pi) {
2678                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2679                         continue;
2680
2681                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2682                         printf("Please remove port %d from forwarding configuration.\n", pi);
2683                         continue;
2684                 }
2685
2686                 if (port_is_bonding_slave(pi)) {
2687                         printf("Please remove port %d from bonded device.\n", pi);
2688                         continue;
2689                 }
2690
2691                 port = &ports[pi];
2692                 if (rte_atomic16_cmpset(&(port->port_status),
2693                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2694                         printf("Port %d is already closed\n", pi);
2695                         continue;
2696                 }
2697
2698                 if (rte_atomic16_cmpset(&(port->port_status),
2699                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2700                         printf("Port %d is now not stopped\n", pi);
2701                         continue;
2702                 }
2703
2704                 if (port->flow_list)
2705                         port_flow_flush(pi);
2706                 rte_eth_dev_close(pi);
2707
2708                 remove_invalid_ports();
2709
2710                 if (rte_atomic16_cmpset(&(port->port_status),
2711                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2712                         printf("Port %d cannot be set to closed\n", pi);
2713         }
2714
2715         printf("Done\n");
2716 }
2717
2718 void
2719 reset_port(portid_t pid)
2720 {
2721         int diag;
2722         portid_t pi;
2723         struct rte_port *port;
2724
2725         if (port_id_is_invalid(pid, ENABLED_WARN))
2726                 return;
2727
2728         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2729                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2730                 printf("Can not reset port(s), please stop port(s) first.\n");
2731                 return;
2732         }
2733
2734         printf("Resetting ports...\n");
2735
2736         RTE_ETH_FOREACH_DEV(pi) {
2737                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2738                         continue;
2739
2740                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2741                         printf("Please remove port %d from forwarding "
2742                                "configuration.\n", pi);
2743                         continue;
2744                 }
2745
2746                 if (port_is_bonding_slave(pi)) {
2747                         printf("Please remove port %d from bonded device.\n",
2748                                pi);
2749                         continue;
2750                 }
2751
2752                 diag = rte_eth_dev_reset(pi);
2753                 if (diag == 0) {
2754                         port = &ports[pi];
2755                         port->need_reconfig = 1;
2756                         port->need_reconfig_queues = 1;
2757                 } else {
2758                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2759                 }
2760         }
2761
2762         printf("Done\n");
2763 }
2764
2765 void
2766 attach_port(char *identifier)
2767 {
2768         portid_t pi;
2769         struct rte_dev_iterator iterator;
2770
2771         printf("Attaching a new port...\n");
2772
2773         if (identifier == NULL) {
2774                 printf("Invalid parameters are specified\n");
2775                 return;
2776         }
2777
2778         if (rte_dev_probe(identifier) < 0) {
2779                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2780                 return;
2781         }
2782
2783         /* first attach mode: event */
2784         if (setup_on_probe_event) {
2785                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2786                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2787                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2788                                         ports[pi].need_setup != 0)
2789                                 setup_attached_port(pi);
2790                 return;
2791         }
2792
2793         /* second attach mode: iterator */
2794         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2795                 /* setup ports matching the devargs used for probing */
2796                 if (port_is_forwarding(pi))
2797                         continue; /* port was already attached before */
2798                 setup_attached_port(pi);
2799         }
2800 }
2801
2802 static void
2803 setup_attached_port(portid_t pi)
2804 {
2805         unsigned int socket_id;
2806         int ret;
2807
2808         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2809         /* if socket_id is invalid, set to the first available socket. */
2810         if (check_socket_id(socket_id) < 0)
2811                 socket_id = socket_ids[0];
2812         reconfig(pi, socket_id);
2813         ret = rte_eth_promiscuous_enable(pi);
2814         if (ret != 0)
2815                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2816                         pi, rte_strerror(-ret));
2817
2818         ports_ids[nb_ports++] = pi;
2819         fwd_ports_ids[nb_fwd_ports++] = pi;
2820         nb_cfg_ports = nb_fwd_ports;
2821         ports[pi].need_setup = 0;
2822         ports[pi].port_status = RTE_PORT_STOPPED;
2823
2824         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2825         printf("Done\n");
2826 }
2827
2828 static void
2829 detach_device(struct rte_device *dev)
2830 {
2831         portid_t sibling;
2832
2833         if (dev == NULL) {
2834                 printf("Device already removed\n");
2835                 return;
2836         }
2837
2838         printf("Removing a device...\n");
2839
2840         if (rte_dev_remove(dev) < 0) {
2841                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2842                 return;
2843         }
2844         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2845                 /* reset mapping between old ports and removed device */
2846                 rte_eth_devices[sibling].device = NULL;
2847                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2848                         /* sibling ports are forced to be closed */
2849                         ports[sibling].port_status = RTE_PORT_CLOSED;
2850                         printf("Port %u is closed\n", sibling);
2851                 }
2852         }
2853
2854         remove_invalid_ports();
2855
2856         printf("Device is detached\n");
2857         printf("Now total ports is %d\n", nb_ports);
2858         printf("Done\n");
2859         return;
2860 }
2861
2862 void
2863 detach_port_device(portid_t port_id)
2864 {
2865         if (port_id_is_invalid(port_id, ENABLED_WARN))
2866                 return;
2867
2868         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2869                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2870                         printf("Port not stopped\n");
2871                         return;
2872                 }
2873                 printf("Port was not closed\n");
2874                 if (ports[port_id].flow_list)
2875                         port_flow_flush(port_id);
2876         }
2877
2878         detach_device(rte_eth_devices[port_id].device);
2879 }
2880
2881 void
2882 detach_devargs(char *identifier)
2883 {
2884         struct rte_dev_iterator iterator;
2885         struct rte_devargs da;
2886         portid_t port_id;
2887
2888         printf("Removing a device...\n");
2889
2890         memset(&da, 0, sizeof(da));
2891         if (rte_devargs_parsef(&da, "%s", identifier)) {
2892                 printf("cannot parse identifier\n");
2893                 if (da.args)
2894                         free(da.args);
2895                 return;
2896         }
2897
2898         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2899                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2900                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2901                                 printf("Port %u not stopped\n", port_id);
2902                                 rte_eth_iterator_cleanup(&iterator);
2903                                 return;
2904                         }
2905
2906                         /* sibling ports are forced to be closed */
2907                         if (ports[port_id].flow_list)
2908                                 port_flow_flush(port_id);
2909                         ports[port_id].port_status = RTE_PORT_CLOSED;
2910                         printf("Port %u is now closed\n", port_id);
2911                 }
2912         }
2913
2914         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2915                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2916                             da.name, da.bus->name);
2917                 return;
2918         }
2919
2920         remove_invalid_ports();
2921
2922         printf("Device %s is detached\n", identifier);
2923         printf("Now total ports is %d\n", nb_ports);
2924         printf("Done\n");
2925 }
2926
2927 void
2928 pmd_test_exit(void)
2929 {
2930         portid_t pt_id;
2931         int ret;
2932         int i;
2933
2934         if (test_done == 0)
2935                 stop_packet_forwarding();
2936
2937         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2938                 if (mempools[i]) {
2939                         if (mp_alloc_type == MP_ALLOC_ANON)
2940                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2941                                                      NULL);
2942                 }
2943         }
2944         if (ports != NULL) {
2945                 no_link_check = 1;
2946                 RTE_ETH_FOREACH_DEV(pt_id) {
2947                         printf("\nStopping port %d...\n", pt_id);
2948                         fflush(stdout);
2949                         stop_port(pt_id);
2950                 }
2951                 RTE_ETH_FOREACH_DEV(pt_id) {
2952                         printf("\nShutting down port %d...\n", pt_id);
2953                         fflush(stdout);
2954                         close_port(pt_id);
2955                 }
2956         }
2957
2958         if (hot_plug) {
2959                 ret = rte_dev_event_monitor_stop();
2960                 if (ret) {
2961                         RTE_LOG(ERR, EAL,
2962                                 "fail to stop device event monitor.");
2963                         return;
2964                 }
2965
2966                 ret = rte_dev_event_callback_unregister(NULL,
2967                         dev_event_callback, NULL);
2968                 if (ret < 0) {
2969                         RTE_LOG(ERR, EAL,
2970                                 "fail to unregister device event callback.\n");
2971                         return;
2972                 }
2973
2974                 ret = rte_dev_hotplug_handle_disable();
2975                 if (ret) {
2976                         RTE_LOG(ERR, EAL,
2977                                 "fail to disable hotplug handling.\n");
2978                         return;
2979                 }
2980         }
2981         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2982                 if (mempools[i])
2983                         rte_mempool_free(mempools[i]);
2984         }
2985
2986         printf("\nBye...\n");
2987 }
2988
2989 typedef void (*cmd_func_t)(void);
2990 struct pmd_test_command {
2991         const char *cmd_name;
2992         cmd_func_t cmd_func;
2993 };
2994
2995 /* Check the link status of all ports in up to 9s, and print them finally */
2996 static void
2997 check_all_ports_link_status(uint32_t port_mask)
2998 {
2999 #define CHECK_INTERVAL 100 /* 100ms */
3000 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3001         portid_t portid;
3002         uint8_t count, all_ports_up, print_flag = 0;
3003         struct rte_eth_link link;
3004         int ret;
3005         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3006
3007         printf("Checking link statuses...\n");
3008         fflush(stdout);
3009         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3010                 all_ports_up = 1;
3011                 RTE_ETH_FOREACH_DEV(portid) {
3012                         if ((port_mask & (1 << portid)) == 0)
3013                                 continue;
3014                         memset(&link, 0, sizeof(link));
3015                         ret = rte_eth_link_get_nowait(portid, &link);
3016                         if (ret < 0) {
3017                                 all_ports_up = 0;
3018                                 if (print_flag == 1)
3019                                         printf("Port %u link get failed: %s\n",
3020                                                 portid, rte_strerror(-ret));
3021                                 continue;
3022                         }
3023                         /* print link status if flag set */
3024                         if (print_flag == 1) {
3025                                 rte_eth_link_to_str(link_status,
3026                                         sizeof(link_status), &link);
3027                                 printf("Port %d %s\n", portid, link_status);
3028                                 continue;
3029                         }
3030                         /* clear all_ports_up flag if any link down */
3031                         if (link.link_status == ETH_LINK_DOWN) {
3032                                 all_ports_up = 0;
3033                                 break;
3034                         }
3035                 }
3036                 /* after finally printing all link status, get out */
3037                 if (print_flag == 1)
3038                         break;
3039
3040                 if (all_ports_up == 0) {
3041                         fflush(stdout);
3042                         rte_delay_ms(CHECK_INTERVAL);
3043                 }
3044
3045                 /* set the print_flag if all ports up or timeout */
3046                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3047                         print_flag = 1;
3048                 }
3049
3050                 if (lsc_interrupt)
3051                         break;
3052         }
3053 }
3054
3055 /*
3056  * This callback is for remove a port for a device. It has limitation because
3057  * it is not for multiple port removal for a device.
3058  * TODO: the device detach invoke will plan to be removed from user side to
3059  * eal. And convert all PMDs to free port resources on ether device closing.
3060  */
3061 static void
3062 rmv_port_callback(void *arg)
3063 {
3064         int need_to_start = 0;
3065         int org_no_link_check = no_link_check;
3066         portid_t port_id = (intptr_t)arg;
3067         struct rte_device *dev;
3068
3069         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3070
3071         if (!test_done && port_is_forwarding(port_id)) {
3072                 need_to_start = 1;
3073                 stop_packet_forwarding();
3074         }
3075         no_link_check = 1;
3076         stop_port(port_id);
3077         no_link_check = org_no_link_check;
3078
3079         /* Save rte_device pointer before closing ethdev port */
3080         dev = rte_eth_devices[port_id].device;
3081         close_port(port_id);
3082         detach_device(dev); /* might be already removed or have more ports */
3083
3084         if (need_to_start)
3085                 start_packet_forwarding(0);
3086 }
3087
3088 /* This function is used by the interrupt thread */
3089 static int
3090 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3091                   void *ret_param)
3092 {
3093         RTE_SET_USED(param);
3094         RTE_SET_USED(ret_param);
3095
3096         if (type >= RTE_ETH_EVENT_MAX) {
3097                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3098                         port_id, __func__, type);
3099                 fflush(stderr);
3100         } else if (event_print_mask & (UINT32_C(1) << type)) {
3101                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3102                         eth_event_desc[type]);
3103                 fflush(stdout);
3104         }
3105
3106         switch (type) {
3107         case RTE_ETH_EVENT_NEW:
3108                 ports[port_id].need_setup = 1;
3109                 ports[port_id].port_status = RTE_PORT_HANDLING;
3110                 break;
3111         case RTE_ETH_EVENT_INTR_RMV:
3112                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3113                         break;
3114                 if (rte_eal_alarm_set(100000,
3115                                 rmv_port_callback, (void *)(intptr_t)port_id))
3116                         fprintf(stderr, "Could not set up deferred device removal\n");
3117                 break;
3118         default:
3119                 break;
3120         }
3121         return 0;
3122 }
3123
3124 static int
3125 register_eth_event_callback(void)
3126 {
3127         int ret;
3128         enum rte_eth_event_type event;
3129
3130         for (event = RTE_ETH_EVENT_UNKNOWN;
3131                         event < RTE_ETH_EVENT_MAX; event++) {
3132                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3133                                 event,
3134                                 eth_event_callback,
3135                                 NULL);
3136                 if (ret != 0) {
3137                         TESTPMD_LOG(ERR, "Failed to register callback for "
3138                                         "%s event\n", eth_event_desc[event]);
3139                         return -1;
3140                 }
3141         }
3142
3143         return 0;
3144 }
3145
3146 /* This function is used by the interrupt thread */
3147 static void
3148 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3149                              __rte_unused void *arg)
3150 {
3151         uint16_t port_id;
3152         int ret;
3153
3154         if (type >= RTE_DEV_EVENT_MAX) {
3155                 fprintf(stderr, "%s called upon invalid event %d\n",
3156                         __func__, type);
3157                 fflush(stderr);
3158         }
3159
3160         switch (type) {
3161         case RTE_DEV_EVENT_REMOVE:
3162                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3163                         device_name);
3164                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3165                 if (ret) {
3166                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3167                                 device_name);
3168                         return;
3169                 }
3170                 /*
3171                  * Because the user's callback is invoked in eal interrupt
3172                  * callback, the interrupt callback need to be finished before
3173                  * it can be unregistered when detaching device. So finish
3174                  * callback soon and use a deferred removal to detach device
3175                  * is need. It is a workaround, once the device detaching be
3176                  * moved into the eal in the future, the deferred removal could
3177                  * be deleted.
3178                  */
3179                 if (rte_eal_alarm_set(100000,
3180                                 rmv_port_callback, (void *)(intptr_t)port_id))
3181                         RTE_LOG(ERR, EAL,
3182                                 "Could not set up deferred device removal\n");
3183                 break;
3184         case RTE_DEV_EVENT_ADD:
3185                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3186                         device_name);
3187                 /* TODO: After finish kernel driver binding,
3188                  * begin to attach port.
3189                  */
3190                 break;
3191         default:
3192                 break;
3193         }
3194 }
3195
3196 static int
3197 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3198 {
3199         uint16_t i;
3200         int diag;
3201         uint8_t mapping_found = 0;
3202
3203         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3204                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3205                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3206                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3207                                         tx_queue_stats_mappings[i].queue_id,
3208                                         tx_queue_stats_mappings[i].stats_counter_id);
3209                         if (diag != 0)
3210                                 return diag;
3211                         mapping_found = 1;
3212                 }
3213         }
3214         if (mapping_found)
3215                 port->tx_queue_stats_mapping_enabled = 1;
3216         return 0;
3217 }
3218
3219 static int
3220 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3221 {
3222         uint16_t i;
3223         int diag;
3224         uint8_t mapping_found = 0;
3225
3226         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3227                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3228                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3229                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3230                                         rx_queue_stats_mappings[i].queue_id,
3231                                         rx_queue_stats_mappings[i].stats_counter_id);
3232                         if (diag != 0)
3233                                 return diag;
3234                         mapping_found = 1;
3235                 }
3236         }
3237         if (mapping_found)
3238                 port->rx_queue_stats_mapping_enabled = 1;
3239         return 0;
3240 }
3241
3242 static void
3243 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3244 {
3245         int diag = 0;
3246
3247         diag = set_tx_queue_stats_mapping_registers(pi, port);
3248         if (diag != 0) {
3249                 if (diag == -ENOTSUP) {
3250                         port->tx_queue_stats_mapping_enabled = 0;
3251                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3252                 }
3253                 else
3254                         rte_exit(EXIT_FAILURE,
3255                                         "set_tx_queue_stats_mapping_registers "
3256                                         "failed for port id=%d diag=%d\n",
3257                                         pi, diag);
3258         }
3259
3260         diag = set_rx_queue_stats_mapping_registers(pi, port);
3261         if (diag != 0) {
3262                 if (diag == -ENOTSUP) {
3263                         port->rx_queue_stats_mapping_enabled = 0;
3264                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3265                 }
3266                 else
3267                         rte_exit(EXIT_FAILURE,
3268                                         "set_rx_queue_stats_mapping_registers "
3269                                         "failed for port id=%d diag=%d\n",
3270                                         pi, diag);
3271         }
3272 }
3273
3274 static void
3275 rxtx_port_config(struct rte_port *port)
3276 {
3277         uint16_t qid;
3278         uint64_t offloads;
3279
3280         for (qid = 0; qid < nb_rxq; qid++) {
3281                 offloads = port->rx_conf[qid].offloads;
3282                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3283                 if (offloads != 0)
3284                         port->rx_conf[qid].offloads = offloads;
3285
3286                 /* Check if any Rx parameters have been passed */
3287                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3288                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3289
3290                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3291                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3292
3293                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3294                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3295
3296                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3297                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3298
3299                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3300                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3301
3302                 port->nb_rx_desc[qid] = nb_rxd;
3303         }
3304
3305         for (qid = 0; qid < nb_txq; qid++) {
3306                 offloads = port->tx_conf[qid].offloads;
3307                 port->tx_conf[qid] = port->dev_info.default_txconf;
3308                 if (offloads != 0)
3309                         port->tx_conf[qid].offloads = offloads;
3310
3311                 /* Check if any Tx parameters have been passed */
3312                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3313                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3314
3315                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3316                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3317
3318                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3319                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3320
3321                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3322                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3323
3324                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3325                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3326
3327                 port->nb_tx_desc[qid] = nb_txd;
3328         }
3329 }
3330
3331 void
3332 init_port_config(void)
3333 {
3334         portid_t pid;
3335         struct rte_port *port;
3336         int ret;
3337
3338         RTE_ETH_FOREACH_DEV(pid) {
3339                 port = &ports[pid];
3340                 port->dev_conf.fdir_conf = fdir_conf;
3341
3342                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3343                 if (ret != 0)
3344                         return;
3345
3346                 if (nb_rxq > 1) {
3347                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3348                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3349                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3350                 } else {
3351                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3352                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3353                 }
3354
3355                 if (port->dcb_flag == 0) {
3356                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3357                                 port->dev_conf.rxmode.mq_mode =
3358                                         (enum rte_eth_rx_mq_mode)
3359                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3360                         else
3361                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3362                 }
3363
3364                 rxtx_port_config(port);
3365
3366                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3367                 if (ret != 0)
3368                         return;
3369
3370                 map_port_queue_stats_mapping_registers(pid, port);
3371 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3372                 rte_pmd_ixgbe_bypass_init(pid);
3373 #endif
3374
3375                 if (lsc_interrupt &&
3376                     (rte_eth_devices[pid].data->dev_flags &
3377                      RTE_ETH_DEV_INTR_LSC))
3378                         port->dev_conf.intr_conf.lsc = 1;
3379                 if (rmv_interrupt &&
3380                     (rte_eth_devices[pid].data->dev_flags &
3381                      RTE_ETH_DEV_INTR_RMV))
3382                         port->dev_conf.intr_conf.rmv = 1;
3383         }
3384 }
3385
3386 void set_port_slave_flag(portid_t slave_pid)
3387 {
3388         struct rte_port *port;
3389
3390         port = &ports[slave_pid];
3391         port->slave_flag = 1;
3392 }
3393
3394 void clear_port_slave_flag(portid_t slave_pid)
3395 {
3396         struct rte_port *port;
3397
3398         port = &ports[slave_pid];
3399         port->slave_flag = 0;
3400 }
3401
3402 uint8_t port_is_bonding_slave(portid_t slave_pid)
3403 {
3404         struct rte_port *port;
3405
3406         port = &ports[slave_pid];
3407         if ((rte_eth_devices[slave_pid].data->dev_flags &
3408             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3409                 return 1;
3410         return 0;
3411 }
3412
3413 const uint16_t vlan_tags[] = {
3414                 0,  1,  2,  3,  4,  5,  6,  7,
3415                 8,  9, 10, 11,  12, 13, 14, 15,
3416                 16, 17, 18, 19, 20, 21, 22, 23,
3417                 24, 25, 26, 27, 28, 29, 30, 31
3418 };
3419
3420 static  int
3421 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3422                  enum dcb_mode_enable dcb_mode,
3423                  enum rte_eth_nb_tcs num_tcs,
3424                  uint8_t pfc_en)
3425 {
3426         uint8_t i;
3427         int32_t rc;
3428         struct rte_eth_rss_conf rss_conf;
3429
3430         /*
3431          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3432          * given above, and the number of traffic classes available for use.
3433          */
3434         if (dcb_mode == DCB_VT_ENABLED) {
3435                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3436                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3437                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3438                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3439
3440                 /* VMDQ+DCB RX and TX configurations */
3441                 vmdq_rx_conf->enable_default_pool = 0;
3442                 vmdq_rx_conf->default_pool = 0;
3443                 vmdq_rx_conf->nb_queue_pools =
3444                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3445                 vmdq_tx_conf->nb_queue_pools =
3446                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3447
3448                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3449                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3450                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3451                         vmdq_rx_conf->pool_map[i].pools =
3452                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3453                 }
3454                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3455                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3456                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3457                 }
3458
3459                 /* set DCB mode of RX and TX of multiple queues */
3460                 eth_conf->rxmode.mq_mode =
3461                                 (enum rte_eth_rx_mq_mode)
3462                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3463                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3464         } else {
3465                 struct rte_eth_dcb_rx_conf *rx_conf =
3466                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3467                 struct rte_eth_dcb_tx_conf *tx_conf =
3468                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3469
3470                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3471
3472                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3473                 if (rc != 0)
3474                         return rc;
3475
3476                 rx_conf->nb_tcs = num_tcs;
3477                 tx_conf->nb_tcs = num_tcs;
3478
3479                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3480                         rx_conf->dcb_tc[i] = i % num_tcs;
3481                         tx_conf->dcb_tc[i] = i % num_tcs;
3482                 }
3483
3484                 eth_conf->rxmode.mq_mode =
3485                                 (enum rte_eth_rx_mq_mode)
3486                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3487                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3488                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3489         }
3490
3491         if (pfc_en)
3492                 eth_conf->dcb_capability_en =
3493                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3494         else
3495                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3496
3497         return 0;
3498 }
3499
3500 int
3501 init_port_dcb_config(portid_t pid,
3502                      enum dcb_mode_enable dcb_mode,
3503                      enum rte_eth_nb_tcs num_tcs,
3504                      uint8_t pfc_en)
3505 {
3506         struct rte_eth_conf port_conf;
3507         struct rte_port *rte_port;
3508         int retval;
3509         uint16_t i;
3510
3511         rte_port = &ports[pid];
3512
3513         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3514         /* Enter DCB configuration status */
3515         dcb_config = 1;
3516
3517         port_conf.rxmode = rte_port->dev_conf.rxmode;
3518         port_conf.txmode = rte_port->dev_conf.txmode;
3519
3520         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3521         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3522         if (retval < 0)
3523                 return retval;
3524         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3525
3526         /* re-configure the device . */
3527         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3528         if (retval < 0)
3529                 return retval;
3530
3531         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3532         if (retval != 0)
3533                 return retval;
3534
3535         /* If dev_info.vmdq_pool_base is greater than 0,
3536          * the queue id of vmdq pools is started after pf queues.
3537          */
3538         if (dcb_mode == DCB_VT_ENABLED &&
3539             rte_port->dev_info.vmdq_pool_base > 0) {
3540                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3541                         " for port %d.", pid);
3542                 return -1;
3543         }
3544
3545         /* Assume the ports in testpmd have the same dcb capability
3546          * and has the same number of rxq and txq in dcb mode
3547          */
3548         if (dcb_mode == DCB_VT_ENABLED) {
3549                 if (rte_port->dev_info.max_vfs > 0) {
3550                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3551                         nb_txq = rte_port->dev_info.nb_tx_queues;
3552                 } else {
3553                         nb_rxq = rte_port->dev_info.max_rx_queues;
3554                         nb_txq = rte_port->dev_info.max_tx_queues;
3555                 }
3556         } else {
3557                 /*if vt is disabled, use all pf queues */
3558                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3559                         nb_rxq = rte_port->dev_info.max_rx_queues;
3560                         nb_txq = rte_port->dev_info.max_tx_queues;
3561                 } else {
3562                         nb_rxq = (queueid_t)num_tcs;
3563                         nb_txq = (queueid_t)num_tcs;
3564
3565                 }
3566         }
3567         rx_free_thresh = 64;
3568
3569         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3570
3571         rxtx_port_config(rte_port);
3572         /* VLAN filter */
3573         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3574         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3575                 rx_vft_set(pid, vlan_tags[i], 1);
3576
3577         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3578         if (retval != 0)
3579                 return retval;
3580
3581         map_port_queue_stats_mapping_registers(pid, rte_port);
3582
3583         rte_port->dcb_flag = 1;
3584
3585         return 0;
3586 }
3587
3588 static void
3589 init_port(void)
3590 {
3591         /* Configuration of Ethernet ports. */
3592         ports = rte_zmalloc("testpmd: ports",
3593                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3594                             RTE_CACHE_LINE_SIZE);
3595         if (ports == NULL) {
3596                 rte_exit(EXIT_FAILURE,
3597                                 "rte_zmalloc(%d struct rte_port) failed\n",
3598                                 RTE_MAX_ETHPORTS);
3599         }
3600
3601         /* Initialize ports NUMA structures */
3602         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3603         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3604         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3605 }
3606
3607 static void
3608 force_quit(void)
3609 {
3610         pmd_test_exit();
3611         prompt_exit();
3612 }
3613
3614 static void
3615 print_stats(void)
3616 {
3617         uint8_t i;
3618         const char clr[] = { 27, '[', '2', 'J', '\0' };
3619         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3620
3621         /* Clear screen and move to top left */
3622         printf("%s%s", clr, top_left);
3623
3624         printf("\nPort statistics ====================================");
3625         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3626                 nic_stats_display(fwd_ports_ids[i]);
3627
3628         fflush(stdout);
3629 }
3630
3631 static void
3632 signal_handler(int signum)
3633 {
3634         if (signum == SIGINT || signum == SIGTERM) {
3635                 printf("\nSignal %d received, preparing to exit...\n",
3636                                 signum);
3637 #ifdef RTE_LIBRTE_PDUMP
3638                 /* uninitialize packet capture framework */
3639                 rte_pdump_uninit();
3640 #endif
3641 #ifdef RTE_LIBRTE_LATENCY_STATS
3642                 if (latencystats_enabled != 0)
3643                         rte_latencystats_uninit();
3644 #endif
3645                 force_quit();
3646                 /* Set flag to indicate the force termination. */
3647                 f_quit = 1;
3648                 /* exit with the expected status */
3649                 signal(signum, SIG_DFL);
3650                 kill(getpid(), signum);
3651         }
3652 }
3653
3654 int
3655 main(int argc, char** argv)
3656 {
3657         int diag;
3658         portid_t port_id;
3659         uint16_t count;
3660         int ret;
3661
3662         signal(SIGINT, signal_handler);
3663         signal(SIGTERM, signal_handler);
3664
3665         testpmd_logtype = rte_log_register("testpmd");
3666         if (testpmd_logtype < 0)
3667                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3668         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3669
3670         diag = rte_eal_init(argc, argv);
3671         if (diag < 0)
3672                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3673                          rte_strerror(rte_errno));
3674
3675         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3676                 rte_exit(EXIT_FAILURE,
3677                          "Secondary process type not supported.\n");
3678
3679         ret = register_eth_event_callback();
3680         if (ret != 0)
3681                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3682
3683 #ifdef RTE_LIBRTE_PDUMP
3684         /* initialize packet capture framework */
3685         rte_pdump_init();
3686 #endif
3687
3688         count = 0;
3689         RTE_ETH_FOREACH_DEV(port_id) {
3690                 ports_ids[count] = port_id;
3691                 count++;
3692         }
3693         nb_ports = (portid_t) count;
3694         if (nb_ports == 0)
3695                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3696
3697         /* allocate port structures, and init them */
3698         init_port();
3699
3700         set_def_fwd_config();
3701         if (nb_lcores == 0)
3702                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3703                          "Check the core mask argument\n");
3704
3705         /* Bitrate/latency stats disabled by default */
3706 #ifdef RTE_LIBRTE_BITRATESTATS
3707         bitrate_enabled = 0;
3708 #endif
3709 #ifdef RTE_LIBRTE_LATENCY_STATS
3710         latencystats_enabled = 0;
3711 #endif
3712
3713         /* on FreeBSD, mlockall() is disabled by default */
3714 #ifdef RTE_EXEC_ENV_FREEBSD
3715         do_mlockall = 0;
3716 #else
3717         do_mlockall = 1;
3718 #endif
3719
3720         argc -= diag;
3721         argv += diag;
3722         if (argc > 1)
3723                 launch_args_parse(argc, argv);
3724
3725         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3726                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3727                         strerror(errno));
3728         }
3729
3730         if (tx_first && interactive)
3731                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3732                                 "interactive mode.\n");
3733
3734         if (tx_first && lsc_interrupt) {
3735                 printf("Warning: lsc_interrupt needs to be off when "
3736                                 " using tx_first. Disabling.\n");
3737                 lsc_interrupt = 0;
3738         }
3739
3740         if (!nb_rxq && !nb_txq)
3741                 printf("Warning: Either rx or tx queues should be non-zero\n");
3742
3743         if (nb_rxq > 1 && nb_rxq > nb_txq)
3744                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3745                        "but nb_txq=%d will prevent to fully test it.\n",
3746                        nb_rxq, nb_txq);
3747
3748         init_config();
3749
3750         if (hot_plug) {
3751                 ret = rte_dev_hotplug_handle_enable();
3752                 if (ret) {
3753                         RTE_LOG(ERR, EAL,
3754                                 "fail to enable hotplug handling.");
3755                         return -1;
3756                 }
3757
3758                 ret = rte_dev_event_monitor_start();
3759                 if (ret) {
3760                         RTE_LOG(ERR, EAL,
3761                                 "fail to start device event monitoring.");
3762                         return -1;
3763                 }
3764
3765                 ret = rte_dev_event_callback_register(NULL,
3766                         dev_event_callback, NULL);
3767                 if (ret) {
3768                         RTE_LOG(ERR, EAL,
3769                                 "fail  to register device event callback\n");
3770                         return -1;
3771                 }
3772         }
3773
3774         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3775                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3776
3777         /* set all ports to promiscuous mode by default */
3778         RTE_ETH_FOREACH_DEV(port_id) {
3779                 ret = rte_eth_promiscuous_enable(port_id);
3780                 if (ret != 0)
3781                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3782                                 port_id, rte_strerror(-ret));
3783         }
3784
3785         /* Init metrics library */
3786         rte_metrics_init(rte_socket_id());
3787
3788 #ifdef RTE_LIBRTE_LATENCY_STATS
3789         if (latencystats_enabled != 0) {
3790                 int ret = rte_latencystats_init(1, NULL);
3791                 if (ret)
3792                         printf("Warning: latencystats init()"
3793                                 " returned error %d\n", ret);
3794                 printf("Latencystats running on lcore %d\n",
3795                         latencystats_lcore_id);
3796         }
3797 #endif
3798
3799         /* Setup bitrate stats */
3800 #ifdef RTE_LIBRTE_BITRATESTATS
3801         if (bitrate_enabled != 0) {
3802                 bitrate_data = rte_stats_bitrate_create();
3803                 if (bitrate_data == NULL)
3804                         rte_exit(EXIT_FAILURE,
3805                                 "Could not allocate bitrate data.\n");
3806                 rte_stats_bitrate_reg(bitrate_data);
3807         }
3808 #endif
3809
3810 #ifdef RTE_LIBRTE_CMDLINE
3811         if (strlen(cmdline_filename) != 0)
3812                 cmdline_read_from_file(cmdline_filename);
3813
3814         if (interactive == 1) {
3815                 if (auto_start) {
3816                         printf("Start automatic packet forwarding\n");
3817                         start_packet_forwarding(0);
3818                 }
3819                 prompt();
3820                 pmd_test_exit();
3821         } else
3822 #endif
3823         {
3824                 char c;
3825                 int rc;
3826
3827                 f_quit = 0;
3828
3829                 printf("No commandline core given, start packet forwarding\n");
3830                 start_packet_forwarding(tx_first);
3831                 if (stats_period != 0) {
3832                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3833                         uint64_t timer_period;
3834
3835                         /* Convert to number of cycles */
3836                         timer_period = stats_period * rte_get_timer_hz();
3837
3838                         while (f_quit == 0) {
3839                                 cur_time = rte_get_timer_cycles();
3840                                 diff_time += cur_time - prev_time;
3841
3842                                 if (diff_time >= timer_period) {
3843                                         print_stats();
3844                                         /* Reset the timer */
3845                                         diff_time = 0;
3846                                 }
3847                                 /* Sleep to avoid unnecessary checks */
3848                                 prev_time = cur_time;
3849                                 sleep(1);
3850                         }
3851                 }
3852
3853                 printf("Press enter to exit\n");
3854                 rc = read(0, &c, 1);
3855                 pmd_test_exit();
3856                 if (rc < 0)
3857                         return 1;
3858         }
3859
3860         ret = rte_eal_cleanup();
3861         if (ret != 0)
3862                 rte_exit(EXIT_FAILURE,
3863                          "EAL cleanup failed: %s\n", strerror(-ret));
3864
3865         return EXIT_SUCCESS;
3866 }