app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 239 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 240 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 241
 242 /*
 243  * Configurable number of RX/TX ring descriptors.
 244  * Defaults are supplied by drivers via ethdev.
 245  */
 246 #define RTE_TEST_RX_DESC_DEFAULT 0
 247 #define RTE_TEST_TX_DESC_DEFAULT 0
 248 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 249 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 250
 251 #define RTE_PMD_PARAM_UNSET -1
 252 /*
 253  * Configurable values of RX and TX ring threshold registers.
 254  */
 255
 256 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 258 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 259
 260 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 262 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 263
 264 /*
 265  * Configurable value of RX free threshold.
 266  */
 267 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 268
 269 /*
 270  * Configurable value of RX drop enable.
 271  */
 272 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 273
 274 /*
 275  * Configurable value of TX free threshold.
 276  */
 277 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 278
 279 /*
 280  * Configurable value of TX RS bit threshold.
 281  */
 282 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 283
 284 /*
 285  * Configurable value of buffered packets before sending.
 286  */
 287 uint16_t noisy_tx_sw_bufsz;
 288
 289 /*
 290  * Configurable value of packet buffer timeout.
 291  */
 292 uint16_t noisy_tx_sw_buf_flush_time;
 293
 294 /*
 295  * Configurable value for size of VNF internal memory area
 296  * used for simulating noisy neighbour behaviour
 297  */
 298 uint64_t noisy_lkup_mem_sz;
 299
 300 /*
 301  * Configurable value of number of random writes done in
 302  * VNF simulation memory area.
 303  */
 304 uint64_t noisy_lkup_num_writes;
 305
 306 /*
 307  * Configurable value of number of random reads done in
 308  * VNF simulation memory area.
 309  */
 310 uint64_t noisy_lkup_num_reads;
 311
 312 /*
 313  * Configurable value of number of random reads/writes done in
 314  * VNF simulation memory area.
 315  */
 316 uint64_t noisy_lkup_num_reads_writes;
 317
 318 /*
 319  * Receive Side Scaling (RSS) configuration.
 320  */
 321 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 322
 323 /*
 324  * Port topology configuration
 325  */
 326 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 327
 328 /*
 329  * Avoids to flush all the RX streams before starts forwarding.
 330  */
 331 uint8_t no_flush_rx = 0; /* flush by default */
 332
 333 /*
 334  * Flow API isolated mode.
 335  */
 336 uint8_t flow_isolate_all;
 337
 338 /*
 339  * Avoids to check link status when starting/stopping a port.
 340  */
 341 uint8_t no_link_check = 0; /* check by default */
 342
 343 /*
 344  * Don't automatically start all ports in interactive mode.
 345  */
 346 uint8_t no_device_start = 0;
 347
 348 /*
 349  * Enable link status change notification
 350  */
 351 uint8_t lsc_interrupt = 1; /* enabled by default */
 352
 353 /*
 354  * Enable device removal notification.
 355  */
 356 uint8_t rmv_interrupt = 1; /* enabled by default */
 357
 358 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 359
 360 /* After attach, port setup is called on event or by iterator */
 361 bool setup_on_probe_event = true;
 362
 363 /* Clear ptypes on port initialization. */
 364 uint8_t clear_ptypes = true;
 365
 366 /* Pretty printing of ethdev events */
 367 static const char * const eth_event_desc[] = {
 368         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 369         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 370         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 371         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 372         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 373         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 374         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 375         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 376         [RTE_ETH_EVENT_NEW] = "device probed",
 377         [RTE_ETH_EVENT_DESTROY] = "device released",
 378         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 379         [RTE_ETH_EVENT_MAX] = NULL,
 380 };
 381
 382 /*
 383  * Display or mask ether events
 384  * Default to all events except VF_MBOX
 385  */
 386 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 388                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 389                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 390                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 391                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 392                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 393                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 394 /*
 395  * Decide if all memory are locked for performance.
 396  */
 397 int do_mlockall = 0;
 398
 399 /*
 400  * NIC bypass mode configuration options.
 401  */
 402
 403 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 404 /* The NIC bypass watchdog timeout. */
 405 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 406 #endif
 407
 408
 409 #ifdef RTE_LIBRTE_LATENCY_STATS
 410
 411 /*
 412  * Set when latency stats is enabled in the commandline
 413  */
 414 uint8_t latencystats_enabled;
 415
 416 /*
 417  * Lcore ID to serive latency statistics.
 418  */
 419 lcoreid_t latencystats_lcore_id = -1;
 420
 421 #endif
 422
 423 /*
 424  * Ethernet device configuration.
 425  */
 426 struct rte_eth_rxmode rx_mode = {
 427         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 428                 /**< Default maximum frame length. */
 429 };
 430
 431 struct rte_eth_txmode tx_mode = {
 432         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 433 };
 434
 435 struct rte_fdir_conf fdir_conf = {
 436         .mode = RTE_FDIR_MODE_NONE,
 437         .pballoc = RTE_FDIR_PBALLOC_64K,
 438         .status = RTE_FDIR_REPORT_STATUS,
 439         .mask = {
 440                 .vlan_tci_mask = 0xFFEF,
 441                 .ipv4_mask     = {
 442                         .src_ip = 0xFFFFFFFF,
 443                         .dst_ip = 0xFFFFFFFF,
 444                 },
 445                 .ipv6_mask     = {
 446                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 447                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 448                 },
 449                 .src_port_mask = 0xFFFF,
 450                 .dst_port_mask = 0xFFFF,
 451                 .mac_addr_byte_mask = 0xFF,
 452                 .tunnel_type_mask = 1,
 453                 .tunnel_id_mask = 0xFFFFFFFF,
 454         },
 455         .drop_queue = 127,
 456 };
 457
 458 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 459
 460 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 461 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 462
 463 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 464 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 465
 466 uint16_t nb_tx_queue_stats_mappings = 0;
 467 uint16_t nb_rx_queue_stats_mappings = 0;
 468
 469 /*
 470  * Display zero values by default for xstats
 471  */
 472 uint8_t xstats_hide_zero;
 473
 474 unsigned int num_sockets = 0;
 475 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 476
 477 #ifdef RTE_LIBRTE_BITRATE
 478 /* Bitrate statistics */
 479 struct rte_stats_bitrates *bitrate_data;
 480 lcoreid_t bitrate_lcore_id;
 481 uint8_t bitrate_enabled;
 482 #endif
 483
 484 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 485 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 486
 487 /* Forward function declarations */
 488 static void setup_attached_port(portid_t pi);
 489 static void map_port_queue_stats_mapping_registers(portid_t pi,
 490                                                    struct rte_port *port);
 491 static void check_all_ports_link_status(uint32_t port_mask);
 492 static int eth_event_callback(portid_t port_id,
 493                               enum rte_eth_event_type type,
 494                               void *param, void *ret_param);
 495 static void dev_event_callback(const char *device_name,
 496                                 enum rte_dev_event_type type,
 497                                 void *param);
 498
 499 /*
 500  * Check if all the ports are started.
 501  * If yes, return positive value. If not, return zero.
 502  */
 503 static int all_ports_started(void);
 504
 505 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 506 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 507
 508 /* Holds the registered mbuf dynamic flags names. */
 509 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 510
 511 /*
 512  * Helper function to check if socket is already discovered.
 513  * If yes, return positive value. If not, return zero.
 514  */
 515 int
 516 new_socket_id(unsigned int socket_id)
 517 {
 518         unsigned int i;
 519
 520         for (i = 0; i < num_sockets; i++) {
 521                 if (socket_ids[i] == socket_id)
 522                         return 0;
 523         }
 524         return 1;
 525 }
 526
 527 /*
 528  * Setup default configuration.
 529  */
 530 static void
 531 set_default_fwd_lcores_config(void)
 532 {
 533         unsigned int i;
 534         unsigned int nb_lc;
 535         unsigned int sock_num;
 536
 537         nb_lc = 0;
 538         for (i = 0; i < RTE_MAX_LCORE; i++) {
 539                 if (!rte_lcore_is_enabled(i))
 540                         continue;
 541                 sock_num = rte_lcore_to_socket_id(i);
 542                 if (new_socket_id(sock_num)) {
 543                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 544                                 rte_exit(EXIT_FAILURE,
 545                                          "Total sockets greater than %u\n",
 546                                          RTE_MAX_NUMA_NODES);
 547                         }
 548                         socket_ids[num_sockets++] = sock_num;
 549                 }
 550                 if (i == rte_get_master_lcore())
 551                         continue;
 552                 fwd_lcores_cpuids[nb_lc++] = i;
 553         }
 554         nb_lcores = (lcoreid_t) nb_lc;
 555         nb_cfg_lcores = nb_lcores;
 556         nb_fwd_lcores = 1;
 557 }
 558
 559 static void
 560 set_def_peer_eth_addrs(void)
 561 {
 562         portid_t i;
 563
 564         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 565                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 566                 peer_eth_addrs[i].addr_bytes[5] = i;
 567         }
 568 }
 569
 570 static void
 571 set_default_fwd_ports_config(void)
 572 {
 573         portid_t pt_id;
 574         int i = 0;
 575
 576         RTE_ETH_FOREACH_DEV(pt_id) {
 577                 fwd_ports_ids[i++] = pt_id;
 578
 579                 /* Update sockets info according to the attached device */
 580                 int socket_id = rte_eth_dev_socket_id(pt_id);
 581                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 582                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 583                                 rte_exit(EXIT_FAILURE,
 584                                          "Total sockets greater than %u\n",
 585                                          RTE_MAX_NUMA_NODES);
 586                         }
 587                         socket_ids[num_sockets++] = socket_id;
 588                 }
 589         }
 590
 591         nb_cfg_ports = nb_ports;
 592         nb_fwd_ports = nb_ports;
 593 }
 594
 595 void
 596 set_def_fwd_config(void)
 597 {
 598         set_default_fwd_lcores_config();
 599         set_def_peer_eth_addrs();
 600         set_default_fwd_ports_config();
 601 }
 602
 603 /* extremely pessimistic estimation of memory required to create a mempool */
 604 static int
 605 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 606 {
 607         unsigned int n_pages, mbuf_per_pg, leftover;
 608         uint64_t total_mem, mbuf_mem, obj_sz;
 609
 610         /* there is no good way to predict how much space the mempool will
 611          * occupy because it will allocate chunks on the fly, and some of those
 612          * will come from default DPDK memory while some will come from our
 613          * external memory, so just assume 128MB will be enough for everyone.
 614          */
 615         uint64_t hdr_mem = 128 << 20;
 616
 617         /* account for possible non-contiguousness */
 618         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 619         if (obj_sz > pgsz) {
 620                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 621                 return -1;
 622         }
 623
 624         mbuf_per_pg = pgsz / obj_sz;
 625         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 626         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 627
 628         mbuf_mem = n_pages * pgsz;
 629
 630         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 631
 632         if (total_mem > SIZE_MAX) {
 633                 TESTPMD_LOG(ERR, "Memory size too big\n");
 634                 return -1;
 635         }
 636         *out = (size_t)total_mem;
 637
 638         return 0;
 639 }
 640
 641 static int
 642 pagesz_flags(uint64_t page_sz)
 643 {
 644         /* as per mmap() manpage, all page sizes are log2 of page size
 645          * shifted by MAP_HUGE_SHIFT
 646          */
 647         int log2 = rte_log2_u64(page_sz);
 648
 649         return (log2 << HUGE_SHIFT);
 650 }
 651
 652 static void *
 653 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 654 {
 655         void *addr;
 656         int flags;
 657
 658         /* allocate anonymous hugepages */
 659         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 660         if (huge)
 661                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 662
 663         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 664         if (addr == MAP_FAILED)
 665                 return NULL;
 666
 667         return addr;
 668 }
 669
 670 struct extmem_param {
 671         void *addr;
 672         size_t len;
 673         size_t pgsz;
 674         rte_iova_t *iova_table;
 675         unsigned int iova_table_len;
 676 };
 677
 678 static int
 679 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 680                 bool huge)
 681 {
 682         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 683                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 684         unsigned int cur_page, n_pages, pgsz_idx;
 685         size_t mem_sz, cur_pgsz;
 686         rte_iova_t *iovas = NULL;
 687         void *addr;
 688         int ret;
 689
 690         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 691                 /* skip anything that is too big */
 692                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 693                         continue;
 694
 695                 cur_pgsz = pgsizes[pgsz_idx];
 696
 697                 /* if we were told not to allocate hugepages, override */
 698                 if (!huge)
 699                         cur_pgsz = sysconf(_SC_PAGESIZE);
 700
 701                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 702                 if (ret < 0) {
 703                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 704                         return -1;
 705                 }
 706
 707                 /* allocate our memory */
 708                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 709
 710                 /* if we couldn't allocate memory with a specified page size,
 711                  * that doesn't mean we can't do it with other page sizes, so
 712                  * try another one.
 713                  */
 714                 if (addr == NULL)
 715                         continue;
 716
 717                 /* store IOVA addresses for every page in this memory area */
 718                 n_pages = mem_sz / cur_pgsz;
 719
 720                 iovas = malloc(sizeof(*iovas) * n_pages);
 721
 722                 if (iovas == NULL) {
 723                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 724                         goto fail;
 725                 }
 726                 /* lock memory if it's not huge pages */
 727                 if (!huge)
 728                         mlock(addr, mem_sz);
 729
 730                 /* populate IOVA addresses */
 731                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 732                         rte_iova_t iova;
 733                         size_t offset;
 734                         void *cur;
 735
 736                         offset = cur_pgsz * cur_page;
 737                         cur = RTE_PTR_ADD(addr, offset);
 738
 739                         /* touch the page before getting its IOVA */
 740                         *(volatile char *)cur = 0;
 741
 742                         iova = rte_mem_virt2iova(cur);
 743
 744                         iovas[cur_page] = iova;
 745                 }
 746
 747                 break;
 748         }
 749         /* if we couldn't allocate anything */
 750         if (iovas == NULL)
 751                 return -1;
 752
 753         param->addr = addr;
 754         param->len = mem_sz;
 755         param->pgsz = cur_pgsz;
 756         param->iova_table = iovas;
 757         param->iova_table_len = n_pages;
 758
 759         return 0;
 760 fail:
 761         if (iovas)
 762                 free(iovas);
 763         if (addr)
 764                 munmap(addr, mem_sz);
 765
 766         return -1;
 767 }
 768
 769 static int
 770 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 771 {
 772         struct extmem_param param;
 773         int socket_id, ret;
 774
 775         memset(&param, 0, sizeof(param));
 776
 777         /* check if our heap exists */
 778         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 779         if (socket_id < 0) {
 780                 /* create our heap */
 781                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 782                 if (ret < 0) {
 783                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 784                         return -1;
 785                 }
 786         }
 787
 788         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 789         if (ret < 0) {
 790                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 791                 return -1;
 792         }
 793
 794         /* we now have a valid memory area, so add it to heap */
 795         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 796                         param.addr, param.len, param.iova_table,
 797                         param.iova_table_len, param.pgsz);
 798
 799         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 800
 801         /* not needed any more */
 802         free(param.iova_table);
 803
 804         if (ret < 0) {
 805                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 806                 munmap(param.addr, param.len);
 807                 return -1;
 808         }
 809
 810         /* success */
 811
 812         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 813                         param.len >> 20);
 814
 815         return 0;
 816 }
 817 static void
 818 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 819              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 820 {
 821         uint16_t pid = 0;
 822         int ret;
 823
 824         RTE_ETH_FOREACH_DEV(pid) {
 825                 struct rte_eth_dev *dev =
 826                         &rte_eth_devices[pid];
 827
 828                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 829                                         memhdr->len);
 830                 if (ret) {
 831                         TESTPMD_LOG(DEBUG,
 832                                     "unable to DMA unmap addr 0x%p "
 833                                     "for device %s\n",
 834                                     memhdr->addr, dev->data->name);
 835                 }
 836         }
 837         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 838         if (ret) {
 839                 TESTPMD_LOG(DEBUG,
 840                             "unable to un-register addr 0x%p\n", memhdr->addr);
 841         }
 842 }
 843
 844 static void
 845 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 846            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 847 {
 848         uint16_t pid = 0;
 849         size_t page_size = sysconf(_SC_PAGESIZE);
 850         int ret;
 851
 852         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 853                                   page_size);
 854         if (ret) {
 855                 TESTPMD_LOG(DEBUG,
 856                             "unable to register addr 0x%p\n", memhdr->addr);
 857                 return;
 858         }
 859         RTE_ETH_FOREACH_DEV(pid) {
 860                 struct rte_eth_dev *dev =
 861                         &rte_eth_devices[pid];
 862
 863                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 864                                       memhdr->len);
 865                 if (ret) {
 866                         TESTPMD_LOG(DEBUG,
 867                                     "unable to DMA map addr 0x%p "
 868                                     "for device %s\n",
 869                                     memhdr->addr, dev->data->name);
 870                 }
 871         }
 872 }
 873
 874 static unsigned int
 875 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 876             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 877 {
 878         struct rte_pktmbuf_extmem *xmem;
 879         unsigned int ext_num, zone_num, elt_num;
 880         uint16_t elt_size;
 881
 882         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 883         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 884         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 885
 886         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 887         if (xmem == NULL) {
 888                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 889                                  "external buffer descriptors\n");
 890                 *ext_mem = NULL;
 891                 return 0;
 892         }
 893         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 894                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 895                 const struct rte_memzone *mz;
 896                 char mz_name[RTE_MEMZONE_NAMESIZE];
 897                 int ret;
 898
 899                 ret = snprintf(mz_name, sizeof(mz_name),
 900                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 901                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 902                         errno = ENAMETOOLONG;
 903                         ext_num = 0;
 904                         break;
 905                 }
 906                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 907                                                  socket_id,
 908                                                  RTE_MEMZONE_IOVA_CONTIG |
 909                                                  RTE_MEMZONE_1GB |
 910                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 911                                                  EXTBUF_ZONE_SIZE);
 912                 if (mz == NULL) {
 913                         /*
 914                          * The caller exits on external buffer creation
 915                          * error, so there is no need to free memzones.
 916                          */
 917                         errno = ENOMEM;
 918                         ext_num = 0;
 919                         break;
 920                 }
 921                 xseg->buf_ptr = mz->addr;
 922                 xseg->buf_iova = mz->iova;
 923                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 924                 xseg->elt_size = elt_size;
 925         }
 926         if (ext_num == 0 && xmem != NULL) {
 927                 free(xmem);
 928                 xmem = NULL;
 929         }
 930         *ext_mem = xmem;
 931         return ext_num;
 932 }
 933
 934 /*
 935  * Configuration initialisation done once at init time.
 936  */
 937 static struct rte_mempool *
 938 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 939                  unsigned int socket_id)
 940 {
 941         char pool_name[RTE_MEMPOOL_NAMESIZE];
 942         struct rte_mempool *rte_mp = NULL;
 943         uint32_t mb_size;
 944
 945         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 946         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 947
 948         TESTPMD_LOG(INFO,
 949                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 950                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 951
 952         switch (mp_alloc_type) {
 953         case MP_ALLOC_NATIVE:
 954                 {
 955                         /* wrapper to rte_mempool_create() */
 956                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 957                                         rte_mbuf_best_mempool_ops());
 958                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 959                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 960                         break;
 961                 }
 962         case MP_ALLOC_ANON:
 963                 {
 964                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 965                                 mb_size, (unsigned int) mb_mempool_cache,
 966                                 sizeof(struct rte_pktmbuf_pool_private),
 967                                 socket_id, mempool_flags);
 968                         if (rte_mp == NULL)
 969                                 goto err;
 970
 971                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 972                                 rte_mempool_free(rte_mp);
 973                                 rte_mp = NULL;
 974                                 goto err;
 975                         }
 976                         rte_pktmbuf_pool_init(rte_mp, NULL);
 977                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 978                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 979                         break;
 980                 }
 981         case MP_ALLOC_XMEM:
 982         case MP_ALLOC_XMEM_HUGE:
 983                 {
 984                         int heap_socket;
 985                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 986
 987                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 988                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 989
 990                         heap_socket =
 991                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 992                         if (heap_socket < 0)
 993                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 994
 995                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 996                                         rte_mbuf_best_mempool_ops());
 997                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 998                                         mb_mempool_cache, 0, mbuf_seg_size,
 999                                         heap_socket);
1000                         break;
1001                 }
1002         case MP_ALLOC_XBUF:
1003                 {
1004                         struct rte_pktmbuf_extmem *ext_mem;
1005                         unsigned int ext_num;
1006
1007                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1008                                                socket_id, pool_name, &ext_mem);
1009                         if (ext_num == 0)
1010                                 rte_exit(EXIT_FAILURE,
1011                                          "Can't create pinned data buffers\n");
1012
1013                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1014                                         rte_mbuf_best_mempool_ops());
1015                         rte_mp = rte_pktmbuf_pool_create_extbuf
1016                                         (pool_name, nb_mbuf, mb_mempool_cache,
1017                                          0, mbuf_seg_size, socket_id,
1018                                          ext_mem, ext_num);
1019                         free(ext_mem);
1020                         break;
1021                 }
1022         default:
1023                 {
1024                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1025                 }
1026         }
1027
1028 err:
1029         if (rte_mp == NULL) {
1030                 rte_exit(EXIT_FAILURE,
1031                         "Creation of mbuf pool for socket %u failed: %s\n",
1032                         socket_id, rte_strerror(rte_errno));
1033         } else if (verbose_level > 0) {
1034                 rte_mempool_dump(stdout, rte_mp);
1035         }
1036         return rte_mp;
1037 }
1038
1039 /*
1040  * Check given socket id is valid or not with NUMA mode,
1041  * if valid, return 0, else return -1
1042  */
1043 static int
1044 check_socket_id(const unsigned int socket_id)
1045 {
1046         static int warning_once = 0;
1047
1048         if (new_socket_id(socket_id)) {
1049                 if (!warning_once && numa_support)
1050                         printf("Warning: NUMA should be configured manually by"
1051                                " using --port-numa-config and"
1052                                " --ring-numa-config parameters along with"
1053                                " --numa.\n");
1054                 warning_once = 1;
1055                 return -1;
1056         }
1057         return 0;
1058 }
1059
1060 /*
1061  * Get the allowed maximum number of RX queues.
1062  * *pid return the port id which has minimal value of
1063  * max_rx_queues in all ports.
1064  */
1065 queueid_t
1066 get_allowed_max_nb_rxq(portid_t *pid)
1067 {
1068         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1069         bool max_rxq_valid = false;
1070         portid_t pi;
1071         struct rte_eth_dev_info dev_info;
1072
1073         RTE_ETH_FOREACH_DEV(pi) {
1074                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1075                         continue;
1076
1077                 max_rxq_valid = true;
1078                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1079                         allowed_max_rxq = dev_info.max_rx_queues;
1080                         *pid = pi;
1081                 }
1082         }
1083         return max_rxq_valid ? allowed_max_rxq : 0;
1084 }
1085
1086 /*
1087  * Check input rxq is valid or not.
1088  * If input rxq is not greater than any of maximum number
1089  * of RX queues of all ports, it is valid.
1090  * if valid, return 0, else return -1
1091  */
1092 int
1093 check_nb_rxq(queueid_t rxq)
1094 {
1095         queueid_t allowed_max_rxq;
1096         portid_t pid = 0;
1097
1098         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1099         if (rxq > allowed_max_rxq) {
1100                 printf("Fail: input rxq (%u) can't be greater "
1101                        "than max_rx_queues (%u) of port %u\n",
1102                        rxq,
1103                        allowed_max_rxq,
1104                        pid);
1105                 return -1;
1106         }
1107         return 0;
1108 }
1109
1110 /*
1111  * Get the allowed maximum number of TX queues.
1112  * *pid return the port id which has minimal value of
1113  * max_tx_queues in all ports.
1114  */
1115 queueid_t
1116 get_allowed_max_nb_txq(portid_t *pid)
1117 {
1118         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1119         bool max_txq_valid = false;
1120         portid_t pi;
1121         struct rte_eth_dev_info dev_info;
1122
1123         RTE_ETH_FOREACH_DEV(pi) {
1124                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1125                         continue;
1126
1127                 max_txq_valid = true;
1128                 if (dev_info.max_tx_queues < allowed_max_txq) {
1129                         allowed_max_txq = dev_info.max_tx_queues;
1130                         *pid = pi;
1131                 }
1132         }
1133         return max_txq_valid ? allowed_max_txq : 0;
1134 }
1135
1136 /*
1137  * Check input txq is valid or not.
1138  * If input txq is not greater than any of maximum number
1139  * of TX queues of all ports, it is valid.
1140  * if valid, return 0, else return -1
1141  */
1142 int
1143 check_nb_txq(queueid_t txq)
1144 {
1145         queueid_t allowed_max_txq;
1146         portid_t pid = 0;
1147
1148         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1149         if (txq > allowed_max_txq) {
1150                 printf("Fail: input txq (%u) can't be greater "
1151                        "than max_tx_queues (%u) of port %u\n",
1152                        txq,
1153                        allowed_max_txq,
1154                        pid);
1155                 return -1;
1156         }
1157         return 0;
1158 }
1159
1160 /*
1161  * Get the allowed maximum number of RXDs of every rx queue.
1162  * *pid return the port id which has minimal value of
1163  * max_rxd in all queues of all ports.
1164  */
1165 static uint16_t
1166 get_allowed_max_nb_rxd(portid_t *pid)
1167 {
1168         uint16_t allowed_max_rxd = UINT16_MAX;
1169         portid_t pi;
1170         struct rte_eth_dev_info dev_info;
1171
1172         RTE_ETH_FOREACH_DEV(pi) {
1173                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1174                         continue;
1175
1176                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1177                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1178                         *pid = pi;
1179                 }
1180         }
1181         return allowed_max_rxd;
1182 }
1183
1184 /*
1185  * Get the allowed minimal number of RXDs of every rx queue.
1186  * *pid return the port id which has minimal value of
1187  * min_rxd in all queues of all ports.
1188  */
1189 static uint16_t
1190 get_allowed_min_nb_rxd(portid_t *pid)
1191 {
1192         uint16_t allowed_min_rxd = 0;
1193         portid_t pi;
1194         struct rte_eth_dev_info dev_info;
1195
1196         RTE_ETH_FOREACH_DEV(pi) {
1197                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1198                         continue;
1199
1200                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1201                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1202                         *pid = pi;
1203                 }
1204         }
1205
1206         return allowed_min_rxd;
1207 }
1208
1209 /*
1210  * Check input rxd is valid or not.
1211  * If input rxd is not greater than any of maximum number
1212  * of RXDs of every Rx queues and is not less than any of
1213  * minimal number of RXDs of every Rx queues, it is valid.
1214  * if valid, return 0, else return -1
1215  */
1216 int
1217 check_nb_rxd(queueid_t rxd)
1218 {
1219         uint16_t allowed_max_rxd;
1220         uint16_t allowed_min_rxd;
1221         portid_t pid = 0;
1222
1223         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1224         if (rxd > allowed_max_rxd) {
1225                 printf("Fail: input rxd (%u) can't be greater "
1226                        "than max_rxds (%u) of port %u\n",
1227                        rxd,
1228                        allowed_max_rxd,
1229                        pid);
1230                 return -1;
1231         }
1232
1233         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1234         if (rxd < allowed_min_rxd) {
1235                 printf("Fail: input rxd (%u) can't be less "
1236                        "than min_rxds (%u) of port %u\n",
1237                        rxd,
1238                        allowed_min_rxd,
1239                        pid);
1240                 return -1;
1241         }
1242
1243         return 0;
1244 }
1245
1246 /*
1247  * Get the allowed maximum number of TXDs of every rx queues.
1248  * *pid return the port id which has minimal value of
1249  * max_txd in every tx queue.
1250  */
1251 static uint16_t
1252 get_allowed_max_nb_txd(portid_t *pid)
1253 {
1254         uint16_t allowed_max_txd = UINT16_MAX;
1255         portid_t pi;
1256         struct rte_eth_dev_info dev_info;
1257
1258         RTE_ETH_FOREACH_DEV(pi) {
1259                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1260                         continue;
1261
1262                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1263                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1264                         *pid = pi;
1265                 }
1266         }
1267         return allowed_max_txd;
1268 }
1269
1270 /*
1271  * Get the allowed maximum number of TXDs of every tx queues.
1272  * *pid return the port id which has minimal value of
1273  * min_txd in every tx queue.
1274  */
1275 static uint16_t
1276 get_allowed_min_nb_txd(portid_t *pid)
1277 {
1278         uint16_t allowed_min_txd = 0;
1279         portid_t pi;
1280         struct rte_eth_dev_info dev_info;
1281
1282         RTE_ETH_FOREACH_DEV(pi) {
1283                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1284                         continue;
1285
1286                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1287                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1288                         *pid = pi;
1289                 }
1290         }
1291
1292         return allowed_min_txd;
1293 }
1294
1295 /*
1296  * Check input txd is valid or not.
1297  * If input txd is not greater than any of maximum number
1298  * of TXDs of every Rx queues, it is valid.
1299  * if valid, return 0, else return -1
1300  */
1301 int
1302 check_nb_txd(queueid_t txd)
1303 {
1304         uint16_t allowed_max_txd;
1305         uint16_t allowed_min_txd;
1306         portid_t pid = 0;
1307
1308         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1309         if (txd > allowed_max_txd) {
1310                 printf("Fail: input txd (%u) can't be greater "
1311                        "than max_txds (%u) of port %u\n",
1312                        txd,
1313                        allowed_max_txd,
1314                        pid);
1315                 return -1;
1316         }
1317
1318         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1319         if (txd < allowed_min_txd) {
1320                 printf("Fail: input txd (%u) can't be less "
1321                        "than min_txds (%u) of port %u\n",
1322                        txd,
1323                        allowed_min_txd,
1324                        pid);
1325                 return -1;
1326         }
1327         return 0;
1328 }
1329
1330
1331 /*
1332  * Get the allowed maximum number of hairpin queues.
1333  * *pid return the port id which has minimal value of
1334  * max_hairpin_queues in all ports.
1335  */
1336 queueid_t
1337 get_allowed_max_nb_hairpinq(portid_t *pid)
1338 {
1339         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1340         portid_t pi;
1341         struct rte_eth_hairpin_cap cap;
1342
1343         RTE_ETH_FOREACH_DEV(pi) {
1344                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1345                         *pid = pi;
1346                         return 0;
1347                 }
1348                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1349                         allowed_max_hairpinq = cap.max_nb_queues;
1350                         *pid = pi;
1351                 }
1352         }
1353         return allowed_max_hairpinq;
1354 }
1355
1356 /*
1357  * Check input hairpin is valid or not.
1358  * If input hairpin is not greater than any of maximum number
1359  * of hairpin queues of all ports, it is valid.
1360  * if valid, return 0, else return -1
1361  */
1362 int
1363 check_nb_hairpinq(queueid_t hairpinq)
1364 {
1365         queueid_t allowed_max_hairpinq;
1366         portid_t pid = 0;
1367
1368         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1369         if (hairpinq > allowed_max_hairpinq) {
1370                 printf("Fail: input hairpin (%u) can't be greater "
1371                        "than max_hairpin_queues (%u) of port %u\n",
1372                        hairpinq, allowed_max_hairpinq, pid);
1373                 return -1;
1374         }
1375         return 0;
1376 }
1377
1378 static void
1379 init_config(void)
1380 {
1381         portid_t pid;
1382         struct rte_port *port;
1383         struct rte_mempool *mbp;
1384         unsigned int nb_mbuf_per_pool;
1385         lcoreid_t  lc_id;
1386         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1387         struct rte_gro_param gro_param;
1388         uint32_t gso_types;
1389         uint16_t data_size;
1390         bool warning = 0;
1391         int k;
1392         int ret;
1393
1394         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1395
1396         /* Configuration of logical cores. */
1397         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1398                                 sizeof(struct fwd_lcore *) * nb_lcores,
1399                                 RTE_CACHE_LINE_SIZE);
1400         if (fwd_lcores == NULL) {
1401                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1402                                                         "failed\n", nb_lcores);
1403         }
1404         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1405                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1406                                                sizeof(struct fwd_lcore),
1407                                                RTE_CACHE_LINE_SIZE);
1408                 if (fwd_lcores[lc_id] == NULL) {
1409                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1410                                                                 "failed\n");
1411                 }
1412                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1413         }
1414
1415         RTE_ETH_FOREACH_DEV(pid) {
1416                 port = &ports[pid];
1417                 /* Apply default TxRx configuration for all ports */
1418                 port->dev_conf.txmode = tx_mode;
1419                 port->dev_conf.rxmode = rx_mode;
1420
1421                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1422                 if (ret != 0)
1423                         rte_exit(EXIT_FAILURE,
1424                                  "rte_eth_dev_info_get() failed\n");
1425
1426                 if (!(port->dev_info.tx_offload_capa &
1427                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1428                         port->dev_conf.txmode.offloads &=
1429                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1430                 if (numa_support) {
1431                         if (port_numa[pid] != NUMA_NO_CONFIG)
1432                                 port_per_socket[port_numa[pid]]++;
1433                         else {
1434                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1435
1436                                 /*
1437                                  * if socket_id is invalid,
1438                                  * set to the first available socket.
1439                                  */
1440                                 if (check_socket_id(socket_id) < 0)
1441                                         socket_id = socket_ids[0];
1442                                 port_per_socket[socket_id]++;
1443                         }
1444                 }
1445
1446                 /* Apply Rx offloads configuration */
1447                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1448                         port->rx_conf[k].offloads =
1449                                 port->dev_conf.rxmode.offloads;
1450                 /* Apply Tx offloads configuration */
1451                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1452                         port->tx_conf[k].offloads =
1453                                 port->dev_conf.txmode.offloads;
1454
1455                 /* set flag to initialize port/queue */
1456                 port->need_reconfig = 1;
1457                 port->need_reconfig_queues = 1;
1458                 port->tx_metadata = 0;
1459
1460                 /* Check for maximum number of segments per MTU. Accordingly
1461                  * update the mbuf data size.
1462                  */
1463                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1464                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1465                         data_size = rx_mode.max_rx_pkt_len /
1466                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1467
1468                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1469                                                         mbuf_data_size) {
1470                                 mbuf_data_size = data_size +
1471                                                  RTE_PKTMBUF_HEADROOM;
1472                                 warning = 1;
1473                         }
1474                 }
1475         }
1476
1477         if (warning)
1478                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1479                             mbuf_data_size);
1480
1481         /*
1482          * Create pools of mbuf.
1483          * If NUMA support is disabled, create a single pool of mbuf in
1484          * socket 0 memory by default.
1485          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1486          *
1487          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1488          * nb_txd can be configured at run time.
1489          */
1490         if (param_total_num_mbufs)
1491                 nb_mbuf_per_pool = param_total_num_mbufs;
1492         else {
1493                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1494                         (nb_lcores * mb_mempool_cache) +
1495                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1496                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1497         }
1498
1499         if (numa_support) {
1500                 uint8_t i;
1501
1502                 for (i = 0; i < num_sockets; i++)
1503                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1504                                                        nb_mbuf_per_pool,
1505                                                        socket_ids[i]);
1506         } else {
1507                 if (socket_num == UMA_NO_CONFIG)
1508                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1509                                                        nb_mbuf_per_pool, 0);
1510                 else
1511                         mempools[socket_num] = mbuf_pool_create
1512                                                         (mbuf_data_size,
1513                                                          nb_mbuf_per_pool,
1514                                                          socket_num);
1515         }
1516
1517         init_port_config();
1518
1519         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1520                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1521         /*
1522          * Records which Mbuf pool to use by each logical core, if needed.
1523          */
1524         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1525                 mbp = mbuf_pool_find(
1526                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1527
1528                 if (mbp == NULL)
1529                         mbp = mbuf_pool_find(0);
1530                 fwd_lcores[lc_id]->mbp = mbp;
1531                 /* initialize GSO context */
1532                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1533                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1534                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1535                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1536                         RTE_ETHER_CRC_LEN;
1537                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1538         }
1539
1540         /* Configuration of packet forwarding streams. */
1541         if (init_fwd_streams() < 0)
1542                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1543
1544         fwd_config_setup();
1545
1546         /* create a gro context for each lcore */
1547         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1548         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1549         gro_param.max_item_per_flow = MAX_PKT_BURST;
1550         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1551                 gro_param.socket_id = rte_lcore_to_socket_id(
1552                                 fwd_lcores_cpuids[lc_id]);
1553                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1554                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1555                         rte_exit(EXIT_FAILURE,
1556                                         "rte_gro_ctx_create() failed\n");
1557                 }
1558         }
1559
1560 #if defined RTE_LIBRTE_PMD_SOFTNIC
1561         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1562                 RTE_ETH_FOREACH_DEV(pid) {
1563                         port = &ports[pid];
1564                         const char *driver = port->dev_info.driver_name;
1565
1566                         if (strcmp(driver, "net_softnic") == 0)
1567                                 port->softport.fwd_lcore_arg = fwd_lcores;
1568                 }
1569         }
1570 #endif
1571
1572 }
1573
1574
1575 void
1576 reconfig(portid_t new_port_id, unsigned socket_id)
1577 {
1578         struct rte_port *port;
1579         int ret;
1580
1581         /* Reconfiguration of Ethernet ports. */
1582         port = &ports[new_port_id];
1583
1584         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1585         if (ret != 0)
1586                 return;
1587
1588         /* set flag to initialize port/queue */
1589         port->need_reconfig = 1;
1590         port->need_reconfig_queues = 1;
1591         port->socket_id = socket_id;
1592
1593         init_port_config();
1594 }
1595
1596
1597 int
1598 init_fwd_streams(void)
1599 {
1600         portid_t pid;
1601         struct rte_port *port;
1602         streamid_t sm_id, nb_fwd_streams_new;
1603         queueid_t q;
1604
1605         /* set socket id according to numa or not */
1606         RTE_ETH_FOREACH_DEV(pid) {
1607                 port = &ports[pid];
1608                 if (nb_rxq > port->dev_info.max_rx_queues) {
1609                         printf("Fail: nb_rxq(%d) is greater than "
1610                                 "max_rx_queues(%d)\n", nb_rxq,
1611                                 port->dev_info.max_rx_queues);
1612                         return -1;
1613                 }
1614                 if (nb_txq > port->dev_info.max_tx_queues) {
1615                         printf("Fail: nb_txq(%d) is greater than "
1616                                 "max_tx_queues(%d)\n", nb_txq,
1617                                 port->dev_info.max_tx_queues);
1618                         return -1;
1619                 }
1620                 if (numa_support) {
1621                         if (port_numa[pid] != NUMA_NO_CONFIG)
1622                                 port->socket_id = port_numa[pid];
1623                         else {
1624                                 port->socket_id = rte_eth_dev_socket_id(pid);
1625
1626                                 /*
1627                                  * if socket_id is invalid,
1628                                  * set to the first available socket.
1629                                  */
1630                                 if (check_socket_id(port->socket_id) < 0)
1631                                         port->socket_id = socket_ids[0];
1632                         }
1633                 }
1634                 else {
1635                         if (socket_num == UMA_NO_CONFIG)
1636                                 port->socket_id = 0;
1637                         else
1638                                 port->socket_id = socket_num;
1639                 }
1640         }
1641
1642         q = RTE_MAX(nb_rxq, nb_txq);
1643         if (q == 0) {
1644                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1645                 return -1;
1646         }
1647         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1648         if (nb_fwd_streams_new == nb_fwd_streams)
1649                 return 0;
1650         /* clear the old */
1651         if (fwd_streams != NULL) {
1652                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1653                         if (fwd_streams[sm_id] == NULL)
1654                                 continue;
1655                         rte_free(fwd_streams[sm_id]);
1656                         fwd_streams[sm_id] = NULL;
1657                 }
1658                 rte_free(fwd_streams);
1659                 fwd_streams = NULL;
1660         }
1661
1662         /* init new */
1663         nb_fwd_streams = nb_fwd_streams_new;
1664         if (nb_fwd_streams) {
1665                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1666                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1667                         RTE_CACHE_LINE_SIZE);
1668                 if (fwd_streams == NULL)
1669                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1670                                  " (struct fwd_stream *)) failed\n",
1671                                  nb_fwd_streams);
1672
1673                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1674                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1675                                 " struct fwd_stream", sizeof(struct fwd_stream),
1676                                 RTE_CACHE_LINE_SIZE);
1677                         if (fwd_streams[sm_id] == NULL)
1678                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1679                                          "(struct fwd_stream) failed\n");
1680                 }
1681         }
1682
1683         return 0;
1684 }
1685
1686 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1687 static void
1688 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1689 {
1690         unsigned int total_burst;
1691         unsigned int nb_burst;
1692         unsigned int burst_stats[3];
1693         uint16_t pktnb_stats[3];
1694         uint16_t nb_pkt;
1695         int burst_percent[3];
1696
1697         /*
1698          * First compute the total number of packet bursts and the
1699          * two highest numbers of bursts of the same number of packets.
1700          */
1701         total_burst = 0;
1702         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1703         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1704         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1705                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1706                 if (nb_burst == 0)
1707                         continue;
1708                 total_burst += nb_burst;
1709                 if (nb_burst > burst_stats[0]) {
1710                         burst_stats[1] = burst_stats[0];
1711                         pktnb_stats[1] = pktnb_stats[0];
1712                         burst_stats[0] = nb_burst;
1713                         pktnb_stats[0] = nb_pkt;
1714                 } else if (nb_burst > burst_stats[1]) {
1715                         burst_stats[1] = nb_burst;
1716                         pktnb_stats[1] = nb_pkt;
1717                 }
1718         }
1719         if (total_burst == 0)
1720                 return;
1721         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1722         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1723                burst_percent[0], (int) pktnb_stats[0]);
1724         if (burst_stats[0] == total_burst) {
1725                 printf("]\n");
1726                 return;
1727         }
1728         if (burst_stats[0] + burst_stats[1] == total_burst) {
1729                 printf(" + %d%% of %d pkts]\n",
1730                        100 - burst_percent[0], pktnb_stats[1]);
1731                 return;
1732         }
1733         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1734         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1735         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1736                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1737                 return;
1738         }
1739         printf(" + %d%% of %d pkts + %d%% of others]\n",
1740                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1741 }
1742 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1743
1744 static void
1745 fwd_stream_stats_display(streamid_t stream_id)
1746 {
1747         struct fwd_stream *fs;
1748         static const char *fwd_top_stats_border = "-------";
1749
1750         fs = fwd_streams[stream_id];
1751         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1752             (fs->fwd_dropped == 0))
1753                 return;
1754         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1755                "TX Port=%2d/Queue=%2d %s\n",
1756                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1757                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1758         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1759                " TX-dropped: %-14"PRIu64,
1760                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1761
1762         /* if checksum mode */
1763         if (cur_fwd_eng == &csum_fwd_engine) {
1764                 printf("  RX- bad IP checksum: %-14"PRIu64
1765                        "  Rx- bad L4 checksum: %-14"PRIu64
1766                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1767                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1768                         fs->rx_bad_outer_l4_csum);
1769         } else {
1770                 printf("\n");
1771         }
1772
1773 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1774         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1775         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1776 #endif
1777 }
1778
1779 void
1780 fwd_stats_display(void)
1781 {
1782         static const char *fwd_stats_border = "----------------------";
1783         static const char *acc_stats_border = "+++++++++++++++";
1784         struct {
1785                 struct fwd_stream *rx_stream;
1786                 struct fwd_stream *tx_stream;
1787                 uint64_t tx_dropped;
1788                 uint64_t rx_bad_ip_csum;
1789                 uint64_t rx_bad_l4_csum;
1790                 uint64_t rx_bad_outer_l4_csum;
1791         } ports_stats[RTE_MAX_ETHPORTS];
1792         uint64_t total_rx_dropped = 0;
1793         uint64_t total_tx_dropped = 0;
1794         uint64_t total_rx_nombuf = 0;
1795         struct rte_eth_stats stats;
1796 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1797         uint64_t fwd_cycles = 0;
1798 #endif
1799         uint64_t total_recv = 0;
1800         uint64_t total_xmit = 0;
1801         struct rte_port *port;
1802         streamid_t sm_id;
1803         portid_t pt_id;
1804         int i;
1805
1806         memset(ports_stats, 0, sizeof(ports_stats));
1807
1808         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1809                 struct fwd_stream *fs = fwd_streams[sm_id];
1810
1811                 if (cur_fwd_config.nb_fwd_streams >
1812                     cur_fwd_config.nb_fwd_ports) {
1813                         fwd_stream_stats_display(sm_id);
1814                 } else {
1815                         ports_stats[fs->tx_port].tx_stream = fs;
1816                         ports_stats[fs->rx_port].rx_stream = fs;
1817                 }
1818
1819                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1820
1821                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1822                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1823                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1824                                 fs->rx_bad_outer_l4_csum;
1825
1826 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1827                 fwd_cycles += fs->core_cycles;
1828 #endif
1829         }
1830         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1831                 uint8_t j;
1832
1833                 pt_id = fwd_ports_ids[i];
1834                 port = &ports[pt_id];
1835
1836                 rte_eth_stats_get(pt_id, &stats);
1837                 stats.ipackets -= port->stats.ipackets;
1838                 stats.opackets -= port->stats.opackets;
1839                 stats.ibytes -= port->stats.ibytes;
1840                 stats.obytes -= port->stats.obytes;
1841                 stats.imissed -= port->stats.imissed;
1842                 stats.oerrors -= port->stats.oerrors;
1843                 stats.rx_nombuf -= port->stats.rx_nombuf;
1844
1845                 total_recv += stats.ipackets;
1846                 total_xmit += stats.opackets;
1847                 total_rx_dropped += stats.imissed;
1848                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1849                 total_tx_dropped += stats.oerrors;
1850                 total_rx_nombuf  += stats.rx_nombuf;
1851
1852                 printf("\n  %s Forward statistics for port %-2d %s\n",
1853                        fwd_stats_border, pt_id, fwd_stats_border);
1854
1855                 if (!port->rx_queue_stats_mapping_enabled &&
1856                     !port->tx_queue_stats_mapping_enabled) {
1857                         printf("  RX-packets: %-14"PRIu64
1858                                " RX-dropped: %-14"PRIu64
1859                                "RX-total: %-"PRIu64"\n",
1860                                stats.ipackets, stats.imissed,
1861                                stats.ipackets + stats.imissed);
1862
1863                         if (cur_fwd_eng == &csum_fwd_engine)
1864                                 printf("  Bad-ipcsum: %-14"PRIu64
1865                                        " Bad-l4csum: %-14"PRIu64
1866                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1867                                        ports_stats[pt_id].rx_bad_ip_csum,
1868                                        ports_stats[pt_id].rx_bad_l4_csum,
1869                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1870                         if (stats.ierrors + stats.rx_nombuf > 0) {
1871                                 printf("  RX-error: %-"PRIu64"\n",
1872                                        stats.ierrors);
1873                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1874                                        stats.rx_nombuf);
1875                         }
1876
1877                         printf("  TX-packets: %-14"PRIu64
1878                                " TX-dropped: %-14"PRIu64
1879                                "TX-total: %-"PRIu64"\n",
1880                                stats.opackets, ports_stats[pt_id].tx_dropped,
1881                                stats.opackets + ports_stats[pt_id].tx_dropped);
1882                 } else {
1883                         printf("  RX-packets:             %14"PRIu64
1884                                "    RX-dropped:%14"PRIu64
1885                                "    RX-total:%14"PRIu64"\n",
1886                                stats.ipackets, stats.imissed,
1887                                stats.ipackets + stats.imissed);
1888
1889                         if (cur_fwd_eng == &csum_fwd_engine)
1890                                 printf("  Bad-ipcsum:%14"PRIu64
1891                                        "    Bad-l4csum:%14"PRIu64
1892                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1893                                        ports_stats[pt_id].rx_bad_ip_csum,
1894                                        ports_stats[pt_id].rx_bad_l4_csum,
1895                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1896                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1897                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1898                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1899                                        stats.rx_nombuf);
1900                         }
1901
1902                         printf("  TX-packets:             %14"PRIu64
1903                                "    TX-dropped:%14"PRIu64
1904                                "    TX-total:%14"PRIu64"\n",
1905                                stats.opackets, ports_stats[pt_id].tx_dropped,
1906                                stats.opackets + ports_stats[pt_id].tx_dropped);
1907                 }
1908
1909 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1910                 if (ports_stats[pt_id].rx_stream)
1911                         pkt_burst_stats_display("RX",
1912                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1913                 if (ports_stats[pt_id].tx_stream)
1914                         pkt_burst_stats_display("TX",
1915                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1916 #endif
1917
1918                 if (port->rx_queue_stats_mapping_enabled) {
1919                         printf("\n");
1920                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1921                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1922                                        "     RX-errors:%14"PRIu64
1923                                        "    RX-bytes:%14"PRIu64"\n",
1924                                        j, stats.q_ipackets[j],
1925                                        stats.q_errors[j], stats.q_ibytes[j]);
1926                         }
1927                         printf("\n");
1928                 }
1929                 if (port->tx_queue_stats_mapping_enabled) {
1930                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1931                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1932                                        "                                 TX-bytes:%14"
1933                                        PRIu64"\n",
1934                                        j, stats.q_opackets[j],
1935                                        stats.q_obytes[j]);
1936                         }
1937                 }
1938
1939                 printf("  %s--------------------------------%s\n",
1940                        fwd_stats_border, fwd_stats_border);
1941         }
1942
1943         printf("\n  %s Accumulated forward statistics for all ports"
1944                "%s\n",
1945                acc_stats_border, acc_stats_border);
1946         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1947                "%-"PRIu64"\n"
1948                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1949                "%-"PRIu64"\n",
1950                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1951                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1952         if (total_rx_nombuf > 0)
1953                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1954         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1955                "%s\n",
1956                acc_stats_border, acc_stats_border);
1957 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1958         if (total_recv > 0)
1959                 printf("\n  CPU cycles/packet=%u (total cycles="
1960                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1961                        (unsigned int)(fwd_cycles / total_recv),
1962                        fwd_cycles, total_recv);
1963 #endif
1964 }
1965
1966 void
1967 fwd_stats_reset(void)
1968 {
1969         streamid_t sm_id;
1970         portid_t pt_id;
1971         int i;
1972
1973         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1974                 pt_id = fwd_ports_ids[i];
1975                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1976         }
1977         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1978                 struct fwd_stream *fs = fwd_streams[sm_id];
1979
1980                 fs->rx_packets = 0;
1981                 fs->tx_packets = 0;
1982                 fs->fwd_dropped = 0;
1983                 fs->rx_bad_ip_csum = 0;
1984                 fs->rx_bad_l4_csum = 0;
1985                 fs->rx_bad_outer_l4_csum = 0;
1986
1987 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1988                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1989                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1990 #endif
1991 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1992                 fs->core_cycles = 0;
1993 #endif
1994         }
1995 }
1996
1997 static void
1998 flush_fwd_rx_queues(void)
1999 {
2000         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2001         portid_t  rxp;
2002         portid_t port_id;
2003         queueid_t rxq;
2004         uint16_t  nb_rx;
2005         uint16_t  i;
2006         uint8_t   j;
2007         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2008         uint64_t timer_period;
2009
2010         /* convert to number of cycles */
2011         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2012
2013         for (j = 0; j < 2; j++) {
2014                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2015                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2016                                 port_id = fwd_ports_ids[rxp];
2017                                 /**
2018                                 * testpmd can stuck in the below do while loop
2019                                 * if rte_eth_rx_burst() always returns nonzero
2020                                 * packets. So timer is added to exit this loop
2021                                 * after 1sec timer expiry.
2022                                 */
2023                                 prev_tsc = rte_rdtsc();
2024                                 do {
2025                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2026                                                 pkts_burst, MAX_PKT_BURST);
2027                                         for (i = 0; i < nb_rx; i++)
2028                                                 rte_pktmbuf_free(pkts_burst[i]);
2029
2030                                         cur_tsc = rte_rdtsc();
2031                                         diff_tsc = cur_tsc - prev_tsc;
2032                                         timer_tsc += diff_tsc;
2033                                 } while ((nb_rx > 0) &&
2034                                         (timer_tsc < timer_period));
2035                                 timer_tsc = 0;
2036                         }
2037                 }
2038                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2039         }
2040 }
2041
2042 static void
2043 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2044 {
2045         struct fwd_stream **fsm;
2046         streamid_t nb_fs;
2047         streamid_t sm_id;
2048 #ifdef RTE_LIBRTE_BITRATE
2049         uint64_t tics_per_1sec;
2050         uint64_t tics_datum;
2051         uint64_t tics_current;
2052         uint16_t i, cnt_ports;
2053
2054         cnt_ports = nb_ports;
2055         tics_datum = rte_rdtsc();
2056         tics_per_1sec = rte_get_timer_hz();
2057 #endif
2058         fsm = &fwd_streams[fc->stream_idx];
2059         nb_fs = fc->stream_nb;
2060         do {
2061                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2062                         (*pkt_fwd)(fsm[sm_id]);
2063 #ifdef RTE_LIBRTE_BITRATE
2064                 if (bitrate_enabled != 0 &&
2065                                 bitrate_lcore_id == rte_lcore_id()) {
2066                         tics_current = rte_rdtsc();
2067                         if (tics_current - tics_datum >= tics_per_1sec) {
2068                                 /* Periodic bitrate calculation */
2069                                 for (i = 0; i < cnt_ports; i++)
2070                                         rte_stats_bitrate_calc(bitrate_data,
2071                                                 ports_ids[i]);
2072                                 tics_datum = tics_current;
2073                         }
2074                 }
2075 #endif
2076 #ifdef RTE_LIBRTE_LATENCY_STATS
2077                 if (latencystats_enabled != 0 &&
2078                                 latencystats_lcore_id == rte_lcore_id())
2079                         rte_latencystats_update();
2080 #endif
2081
2082         } while (! fc->stopped);
2083 }
2084
2085 static int
2086 start_pkt_forward_on_core(void *fwd_arg)
2087 {
2088         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2089                              cur_fwd_config.fwd_eng->packet_fwd);
2090         return 0;
2091 }
2092
2093 /*
2094  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2095  * Used to start communication flows in network loopback test configurations.
2096  */
2097 static int
2098 run_one_txonly_burst_on_core(void *fwd_arg)
2099 {
2100         struct fwd_lcore *fwd_lc;
2101         struct fwd_lcore tmp_lcore;
2102
2103         fwd_lc = (struct fwd_lcore *) fwd_arg;
2104         tmp_lcore = *fwd_lc;
2105         tmp_lcore.stopped = 1;
2106         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2107         return 0;
2108 }
2109
2110 /*
2111  * Launch packet forwarding:
2112  *     - Setup per-port forwarding context.
2113  *     - launch logical cores with their forwarding configuration.
2114  */
2115 static void
2116 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2117 {
2118         port_fwd_begin_t port_fwd_begin;
2119         unsigned int i;
2120         unsigned int lc_id;
2121         int diag;
2122
2123         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2124         if (port_fwd_begin != NULL) {
2125                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2126                         (*port_fwd_begin)(fwd_ports_ids[i]);
2127         }
2128         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2129                 lc_id = fwd_lcores_cpuids[i];
2130                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2131                         fwd_lcores[i]->stopped = 0;
2132                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2133                                                      fwd_lcores[i], lc_id);
2134                         if (diag != 0)
2135                                 printf("launch lcore %u failed - diag=%d\n",
2136                                        lc_id, diag);
2137                 }
2138         }
2139 }
2140
2141 /*
2142  * Launch packet forwarding configuration.
2143  */
2144 void
2145 start_packet_forwarding(int with_tx_first)
2146 {
2147         port_fwd_begin_t port_fwd_begin;
2148         port_fwd_end_t  port_fwd_end;
2149         struct rte_port *port;
2150         unsigned int i;
2151         portid_t   pt_id;
2152
2153         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2154                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2155
2156         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2157                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2158
2159         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2160                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2161                 (!nb_rxq || !nb_txq))
2162                 rte_exit(EXIT_FAILURE,
2163                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2164                         cur_fwd_eng->fwd_mode_name);
2165
2166         if (all_ports_started() == 0) {
2167                 printf("Not all ports were started\n");
2168                 return;
2169         }
2170         if (test_done == 0) {
2171                 printf("Packet forwarding already started\n");
2172                 return;
2173         }
2174
2175
2176         if(dcb_test) {
2177                 for (i = 0; i < nb_fwd_ports; i++) {
2178                         pt_id = fwd_ports_ids[i];
2179                         port = &ports[pt_id];
2180                         if (!port->dcb_flag) {
2181                                 printf("In DCB mode, all forwarding ports must "
2182                                        "be configured in this mode.\n");
2183                                 return;
2184                         }
2185                 }
2186                 if (nb_fwd_lcores == 1) {
2187                         printf("In DCB mode,the nb forwarding cores "
2188                                "should be larger than 1.\n");
2189                         return;
2190                 }
2191         }
2192         test_done = 0;
2193
2194         fwd_config_setup();
2195
2196         if(!no_flush_rx)
2197                 flush_fwd_rx_queues();
2198
2199         pkt_fwd_config_display(&cur_fwd_config);
2200         rxtx_config_display();
2201
2202         fwd_stats_reset();
2203         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2204                 pt_id = fwd_ports_ids[i];
2205                 port = &ports[pt_id];
2206                 map_port_queue_stats_mapping_registers(pt_id, port);
2207         }
2208         if (with_tx_first) {
2209                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2210                 if (port_fwd_begin != NULL) {
2211                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2212                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2213                 }
2214                 while (with_tx_first--) {
2215                         launch_packet_forwarding(
2216                                         run_one_txonly_burst_on_core);
2217                         rte_eal_mp_wait_lcore();
2218                 }
2219                 port_fwd_end = tx_only_engine.port_fwd_end;
2220                 if (port_fwd_end != NULL) {
2221                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2222                                 (*port_fwd_end)(fwd_ports_ids[i]);
2223                 }
2224         }
2225         launch_packet_forwarding(start_pkt_forward_on_core);
2226 }
2227
2228 void
2229 stop_packet_forwarding(void)
2230 {
2231         port_fwd_end_t port_fwd_end;
2232         lcoreid_t lc_id;
2233         portid_t pt_id;
2234         int i;
2235
2236         if (test_done) {
2237                 printf("Packet forwarding not started\n");
2238                 return;
2239         }
2240         printf("Telling cores to stop...");
2241         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2242                 fwd_lcores[lc_id]->stopped = 1;
2243         printf("\nWaiting for lcores to finish...\n");
2244         rte_eal_mp_wait_lcore();
2245         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2246         if (port_fwd_end != NULL) {
2247                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2248                         pt_id = fwd_ports_ids[i];
2249                         (*port_fwd_end)(pt_id);
2250                 }
2251         }
2252
2253         fwd_stats_display();
2254
2255         printf("\nDone.\n");
2256         test_done = 1;
2257 }
2258
2259 void
2260 dev_set_link_up(portid_t pid)
2261 {
2262         if (rte_eth_dev_set_link_up(pid) < 0)
2263                 printf("\nSet link up fail.\n");
2264 }
2265
2266 void
2267 dev_set_link_down(portid_t pid)
2268 {
2269         if (rte_eth_dev_set_link_down(pid) < 0)
2270                 printf("\nSet link down fail.\n");
2271 }
2272
2273 static int
2274 all_ports_started(void)
2275 {
2276         portid_t pi;
2277         struct rte_port *port;
2278
2279         RTE_ETH_FOREACH_DEV(pi) {
2280                 port = &ports[pi];
2281                 /* Check if there is a port which is not started */
2282                 if ((port->port_status != RTE_PORT_STARTED) &&
2283                         (port->slave_flag == 0))
2284                         return 0;
2285         }
2286
2287         /* No port is not started */
2288         return 1;
2289 }
2290
2291 int
2292 port_is_stopped(portid_t port_id)
2293 {
2294         struct rte_port *port = &ports[port_id];
2295
2296         if ((port->port_status != RTE_PORT_STOPPED) &&
2297             (port->slave_flag == 0))
2298                 return 0;
2299         return 1;
2300 }
2301
2302 int
2303 all_ports_stopped(void)
2304 {
2305         portid_t pi;
2306
2307         RTE_ETH_FOREACH_DEV(pi) {
2308                 if (!port_is_stopped(pi))
2309                         return 0;
2310         }
2311
2312         return 1;
2313 }
2314
2315 int
2316 port_is_started(portid_t port_id)
2317 {
2318         if (port_id_is_invalid(port_id, ENABLED_WARN))
2319                 return 0;
2320
2321         if (ports[port_id].port_status != RTE_PORT_STARTED)
2322                 return 0;
2323
2324         return 1;
2325 }
2326
2327 /* Configure the Rx and Tx hairpin queues for the selected port. */
2328 static int
2329 setup_hairpin_queues(portid_t pi)
2330 {
2331         queueid_t qi;
2332         struct rte_eth_hairpin_conf hairpin_conf = {
2333                 .peer_count = 1,
2334         };
2335         int i;
2336         int diag;
2337         struct rte_port *port = &ports[pi];
2338
2339         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2340                 hairpin_conf.peers[0].port = pi;
2341                 hairpin_conf.peers[0].queue = i + nb_rxq;
2342                 diag = rte_eth_tx_hairpin_queue_setup
2343                         (pi, qi, nb_txd, &hairpin_conf);
2344                 i++;
2345                 if (diag == 0)
2346                         continue;
2347
2348                 /* Fail to setup rx queue, return */
2349                 if (rte_atomic16_cmpset(&(port->port_status),
2350                                         RTE_PORT_HANDLING,
2351                                         RTE_PORT_STOPPED) == 0)
2352                         printf("Port %d can not be set back "
2353                                         "to stopped\n", pi);
2354                 printf("Fail to configure port %d hairpin "
2355                                 "queues\n", pi);
2356                 /* try to reconfigure queues next time */
2357                 port->need_reconfig_queues = 1;
2358                 return -1;
2359         }
2360         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2361                 hairpin_conf.peers[0].port = pi;
2362                 hairpin_conf.peers[0].queue = i + nb_txq;
2363                 diag = rte_eth_rx_hairpin_queue_setup
2364                         (pi, qi, nb_rxd, &hairpin_conf);
2365                 i++;
2366                 if (diag == 0)
2367                         continue;
2368
2369                 /* Fail to setup rx queue, return */
2370                 if (rte_atomic16_cmpset(&(port->port_status),
2371                                         RTE_PORT_HANDLING,
2372                                         RTE_PORT_STOPPED) == 0)
2373                         printf("Port %d can not be set back "
2374                                         "to stopped\n", pi);
2375                 printf("Fail to configure port %d hairpin "
2376                                 "queues\n", pi);
2377                 /* try to reconfigure queues next time */
2378                 port->need_reconfig_queues = 1;
2379                 return -1;
2380         }
2381         return 0;
2382 }
2383
2384 int
2385 start_port(portid_t pid)
2386 {
2387         int diag, need_check_link_status = -1;
2388         portid_t pi;
2389         queueid_t qi;
2390         struct rte_port *port;
2391         struct rte_ether_addr mac_addr;
2392         struct rte_eth_hairpin_cap cap;
2393
2394         if (port_id_is_invalid(pid, ENABLED_WARN))
2395                 return 0;
2396
2397         if(dcb_config)
2398                 dcb_test = 1;
2399         RTE_ETH_FOREACH_DEV(pi) {
2400                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2401                         continue;
2402
2403                 need_check_link_status = 0;
2404                 port = &ports[pi];
2405                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2406                                                  RTE_PORT_HANDLING) == 0) {
2407                         printf("Port %d is now not stopped\n", pi);
2408                         continue;
2409                 }
2410
2411                 if (port->need_reconfig > 0) {
2412                         port->need_reconfig = 0;
2413
2414                         if (flow_isolate_all) {
2415                                 int ret = port_flow_isolate(pi, 1);
2416                                 if (ret) {
2417                                         printf("Failed to apply isolated"
2418                                                " mode on port %d\n", pi);
2419                                         return -1;
2420                                 }
2421                         }
2422                         configure_rxtx_dump_callbacks(0);
2423                         printf("Configuring Port %d (socket %u)\n", pi,
2424                                         port->socket_id);
2425                         if (nb_hairpinq > 0 &&
2426                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2427                                 printf("Port %d doesn't support hairpin "
2428                                        "queues\n", pi);
2429                                 return -1;
2430                         }
2431                         /* configure port */
2432                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2433                                                      nb_txq + nb_hairpinq,
2434                                                      &(port->dev_conf));
2435                         if (diag != 0) {
2436                                 if (rte_atomic16_cmpset(&(port->port_status),
2437                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2438                                         printf("Port %d can not be set back "
2439                                                         "to stopped\n", pi);
2440                                 printf("Fail to configure port %d\n", pi);
2441                                 /* try to reconfigure port next time */
2442                                 port->need_reconfig = 1;
2443                                 return -1;
2444                         }
2445                 }
2446                 if (port->need_reconfig_queues > 0) {
2447                         port->need_reconfig_queues = 0;
2448                         /* setup tx queues */
2449                         for (qi = 0; qi < nb_txq; qi++) {
2450                                 if ((numa_support) &&
2451                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2452                                         diag = rte_eth_tx_queue_setup(pi, qi,
2453                                                 port->nb_tx_desc[qi],
2454                                                 txring_numa[pi],
2455                                                 &(port->tx_conf[qi]));
2456                                 else
2457                                         diag = rte_eth_tx_queue_setup(pi, qi,
2458                                                 port->nb_tx_desc[qi],
2459                                                 port->socket_id,
2460                                                 &(port->tx_conf[qi]));
2461
2462                                 if (diag == 0)
2463                                         continue;
2464
2465                                 /* Fail to setup tx queue, return */
2466                                 if (rte_atomic16_cmpset(&(port->port_status),
2467                                                         RTE_PORT_HANDLING,
2468                                                         RTE_PORT_STOPPED) == 0)
2469                                         printf("Port %d can not be set back "
2470                                                         "to stopped\n", pi);
2471                                 printf("Fail to configure port %d tx queues\n",
2472                                        pi);
2473                                 /* try to reconfigure queues next time */
2474                                 port->need_reconfig_queues = 1;
2475                                 return -1;
2476                         }
2477                         for (qi = 0; qi < nb_rxq; qi++) {
2478                                 /* setup rx queues */
2479                                 if ((numa_support) &&
2480                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2481                                         struct rte_mempool * mp =
2482                                                 mbuf_pool_find(rxring_numa[pi]);
2483                                         if (mp == NULL) {
2484                                                 printf("Failed to setup RX queue:"
2485                                                         "No mempool allocation"
2486                                                         " on the socket %d\n",
2487                                                         rxring_numa[pi]);
2488                                                 return -1;
2489                                         }
2490
2491                                         diag = rte_eth_rx_queue_setup(pi, qi,
2492                                              port->nb_rx_desc[qi],
2493                                              rxring_numa[pi],
2494                                              &(port->rx_conf[qi]),
2495                                              mp);
2496                                 } else {
2497                                         struct rte_mempool *mp =
2498                                                 mbuf_pool_find(port->socket_id);
2499                                         if (mp == NULL) {
2500                                                 printf("Failed to setup RX queue:"
2501                                                         "No mempool allocation"
2502                                                         " on the socket %d\n",
2503                                                         port->socket_id);
2504                                                 return -1;
2505                                         }
2506                                         diag = rte_eth_rx_queue_setup(pi, qi,
2507                                              port->nb_rx_desc[qi],
2508                                              port->socket_id,
2509                                              &(port->rx_conf[qi]),
2510                                              mp);
2511                                 }
2512                                 if (diag == 0)
2513                                         continue;
2514
2515                                 /* Fail to setup rx queue, return */
2516                                 if (rte_atomic16_cmpset(&(port->port_status),
2517                                                         RTE_PORT_HANDLING,
2518                                                         RTE_PORT_STOPPED) == 0)
2519                                         printf("Port %d can not be set back "
2520                                                         "to stopped\n", pi);
2521                                 printf("Fail to configure port %d rx queues\n",
2522                                        pi);
2523                                 /* try to reconfigure queues next time */
2524                                 port->need_reconfig_queues = 1;
2525                                 return -1;
2526                         }
2527                         /* setup hairpin queues */
2528                         if (setup_hairpin_queues(pi) != 0)
2529                                 return -1;
2530                 }
2531                 configure_rxtx_dump_callbacks(verbose_level);
2532                 if (clear_ptypes) {
2533                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2534                                         NULL, 0);
2535                         if (diag < 0)
2536                                 printf(
2537                                 "Port %d: Failed to disable Ptype parsing\n",
2538                                 pi);
2539                 }
2540
2541                 /* start port */
2542                 if (rte_eth_dev_start(pi) < 0) {
2543                         printf("Fail to start port %d\n", pi);
2544
2545                         /* Fail to setup rx queue, return */
2546                         if (rte_atomic16_cmpset(&(port->port_status),
2547                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2548                                 printf("Port %d can not be set back to "
2549                                                         "stopped\n", pi);
2550                         continue;
2551                 }
2552
2553                 if (rte_atomic16_cmpset(&(port->port_status),
2554                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2555                         printf("Port %d can not be set into started\n", pi);
2556
2557                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2558                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2559                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2560                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2561                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2562
2563                 /* at least one port started, need checking link status */
2564                 need_check_link_status = 1;
2565         }
2566
2567         if (need_check_link_status == 1 && !no_link_check)
2568                 check_all_ports_link_status(RTE_PORT_ALL);
2569         else if (need_check_link_status == 0)
2570                 printf("Please stop the ports first\n");
2571
2572         printf("Done\n");
2573         return 0;
2574 }
2575
2576 void
2577 stop_port(portid_t pid)
2578 {
2579         portid_t pi;
2580         struct rte_port *port;
2581         int need_check_link_status = 0;
2582
2583         if (dcb_test) {
2584                 dcb_test = 0;
2585                 dcb_config = 0;
2586         }
2587
2588         if (port_id_is_invalid(pid, ENABLED_WARN))
2589                 return;
2590
2591         printf("Stopping ports...\n");
2592
2593         RTE_ETH_FOREACH_DEV(pi) {
2594                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2595                         continue;
2596
2597                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2598                         printf("Please remove port %d from forwarding configuration.\n", pi);
2599                         continue;
2600                 }
2601
2602                 if (port_is_bonding_slave(pi)) {
2603                         printf("Please remove port %d from bonded device.\n", pi);
2604                         continue;
2605                 }
2606
2607                 port = &ports[pi];
2608                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2609                                                 RTE_PORT_HANDLING) == 0)
2610                         continue;
2611
2612                 rte_eth_dev_stop(pi);
2613
2614                 if (rte_atomic16_cmpset(&(port->port_status),
2615                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2616                         printf("Port %d can not be set into stopped\n", pi);
2617                 need_check_link_status = 1;
2618         }
2619         if (need_check_link_status && !no_link_check)
2620                 check_all_ports_link_status(RTE_PORT_ALL);
2621
2622         printf("Done\n");
2623 }
2624
2625 static void
2626 remove_invalid_ports_in(portid_t *array, portid_t *total)
2627 {
2628         portid_t i;
2629         portid_t new_total = 0;
2630
2631         for (i = 0; i < *total; i++)
2632                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2633                         array[new_total] = array[i];
2634                         new_total++;
2635                 }
2636         *total = new_total;
2637 }
2638
2639 static void
2640 remove_invalid_ports(void)
2641 {
2642         remove_invalid_ports_in(ports_ids, &nb_ports);
2643         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2644         nb_cfg_ports = nb_fwd_ports;
2645 }
2646
2647 void
2648 close_port(portid_t pid)
2649 {
2650         portid_t pi;
2651         struct rte_port *port;
2652
2653         if (port_id_is_invalid(pid, ENABLED_WARN))
2654                 return;
2655
2656         printf("Closing ports...\n");
2657
2658         RTE_ETH_FOREACH_DEV(pi) {
2659                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2660                         continue;
2661
2662                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2663                         printf("Please remove port %d from forwarding configuration.\n", pi);
2664                         continue;
2665                 }
2666
2667                 if (port_is_bonding_slave(pi)) {
2668                         printf("Please remove port %d from bonded device.\n", pi);
2669                         continue;
2670                 }
2671
2672                 port = &ports[pi];
2673                 if (rte_atomic16_cmpset(&(port->port_status),
2674                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2675                         printf("Port %d is already closed\n", pi);
2676                         continue;
2677                 }
2678
2679                 if (rte_atomic16_cmpset(&(port->port_status),
2680                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2681                         printf("Port %d is now not stopped\n", pi);
2682                         continue;
2683                 }
2684
2685                 if (port->flow_list)
2686                         port_flow_flush(pi);
2687                 rte_eth_dev_close(pi);
2688
2689                 remove_invalid_ports();
2690
2691                 if (rte_atomic16_cmpset(&(port->port_status),
2692                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2693                         printf("Port %d cannot be set to closed\n", pi);
2694         }
2695
2696         printf("Done\n");
2697 }
2698
2699 void
2700 reset_port(portid_t pid)
2701 {
2702         int diag;
2703         portid_t pi;
2704         struct rte_port *port;
2705
2706         if (port_id_is_invalid(pid, ENABLED_WARN))
2707                 return;
2708
2709         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2710                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2711                 printf("Can not reset port(s), please stop port(s) first.\n");
2712                 return;
2713         }
2714
2715         printf("Resetting ports...\n");
2716
2717         RTE_ETH_FOREACH_DEV(pi) {
2718                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2719                         continue;
2720
2721                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2722                         printf("Please remove port %d from forwarding "
2723                                "configuration.\n", pi);
2724                         continue;
2725                 }
2726
2727                 if (port_is_bonding_slave(pi)) {
2728                         printf("Please remove port %d from bonded device.\n",
2729                                pi);
2730                         continue;
2731                 }
2732
2733                 diag = rte_eth_dev_reset(pi);
2734                 if (diag == 0) {
2735                         port = &ports[pi];
2736                         port->need_reconfig = 1;
2737                         port->need_reconfig_queues = 1;
2738                 } else {
2739                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2740                 }
2741         }
2742
2743         printf("Done\n");
2744 }
2745
2746 void
2747 attach_port(char *identifier)
2748 {
2749         portid_t pi;
2750         struct rte_dev_iterator iterator;
2751
2752         printf("Attaching a new port...\n");
2753
2754         if (identifier == NULL) {
2755                 printf("Invalid parameters are specified\n");
2756                 return;
2757         }
2758
2759         if (rte_dev_probe(identifier) < 0) {
2760                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2761                 return;
2762         }
2763
2764         /* first attach mode: event */
2765         if (setup_on_probe_event) {
2766                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2767                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2768                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2769                                         ports[pi].need_setup != 0)
2770                                 setup_attached_port(pi);
2771                 return;
2772         }
2773
2774         /* second attach mode: iterator */
2775         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2776                 /* setup ports matching the devargs used for probing */
2777                 if (port_is_forwarding(pi))
2778                         continue; /* port was already attached before */
2779                 setup_attached_port(pi);
2780         }
2781 }
2782
2783 static void
2784 setup_attached_port(portid_t pi)
2785 {
2786         unsigned int socket_id;
2787         int ret;
2788
2789         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2790         /* if socket_id is invalid, set to the first available socket. */
2791         if (check_socket_id(socket_id) < 0)
2792                 socket_id = socket_ids[0];
2793         reconfig(pi, socket_id);
2794         ret = rte_eth_promiscuous_enable(pi);
2795         if (ret != 0)
2796                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2797                         pi, rte_strerror(-ret));
2798
2799         ports_ids[nb_ports++] = pi;
2800         fwd_ports_ids[nb_fwd_ports++] = pi;
2801         nb_cfg_ports = nb_fwd_ports;
2802         ports[pi].need_setup = 0;
2803         ports[pi].port_status = RTE_PORT_STOPPED;
2804
2805         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2806         printf("Done\n");
2807 }
2808
2809 static void
2810 detach_device(struct rte_device *dev)
2811 {
2812         portid_t sibling;
2813
2814         if (dev == NULL) {
2815                 printf("Device already removed\n");
2816                 return;
2817         }
2818
2819         printf("Removing a device...\n");
2820
2821         if (rte_dev_remove(dev) < 0) {
2822                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2823                 return;
2824         }
2825         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2826                 /* reset mapping between old ports and removed device */
2827                 rte_eth_devices[sibling].device = NULL;
2828                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2829                         /* sibling ports are forced to be closed */
2830                         ports[sibling].port_status = RTE_PORT_CLOSED;
2831                         printf("Port %u is closed\n", sibling);
2832                 }
2833         }
2834
2835         remove_invalid_ports();
2836
2837         printf("Device is detached\n");
2838         printf("Now total ports is %d\n", nb_ports);
2839         printf("Done\n");
2840         return;
2841 }
2842
2843 void
2844 detach_port_device(portid_t port_id)
2845 {
2846         if (port_id_is_invalid(port_id, ENABLED_WARN))
2847                 return;
2848
2849         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2850                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2851                         printf("Port not stopped\n");
2852                         return;
2853                 }
2854                 printf("Port was not closed\n");
2855                 if (ports[port_id].flow_list)
2856                         port_flow_flush(port_id);
2857         }
2858
2859         detach_device(rte_eth_devices[port_id].device);
2860 }
2861
2862 void
2863 detach_devargs(char *identifier)
2864 {
2865         struct rte_dev_iterator iterator;
2866         struct rte_devargs da;
2867         portid_t port_id;
2868
2869         printf("Removing a device...\n");
2870
2871         memset(&da, 0, sizeof(da));
2872         if (rte_devargs_parsef(&da, "%s", identifier)) {
2873                 printf("cannot parse identifier\n");
2874                 if (da.args)
2875                         free(da.args);
2876                 return;
2877         }
2878
2879         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2880                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2881                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2882                                 printf("Port %u not stopped\n", port_id);
2883                                 rte_eth_iterator_cleanup(&iterator);
2884                                 return;
2885                         }
2886
2887                         /* sibling ports are forced to be closed */
2888                         if (ports[port_id].flow_list)
2889                                 port_flow_flush(port_id);
2890                         ports[port_id].port_status = RTE_PORT_CLOSED;
2891                         printf("Port %u is now closed\n", port_id);
2892                 }
2893         }
2894
2895         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2896                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2897                             da.name, da.bus->name);
2898                 return;
2899         }
2900
2901         remove_invalid_ports();
2902
2903         printf("Device %s is detached\n", identifier);
2904         printf("Now total ports is %d\n", nb_ports);
2905         printf("Done\n");
2906 }
2907
2908 void
2909 pmd_test_exit(void)
2910 {
2911         portid_t pt_id;
2912         int ret;
2913         int i;
2914
2915         if (test_done == 0)
2916                 stop_packet_forwarding();
2917
2918         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2919                 if (mempools[i]) {
2920                         if (mp_alloc_type == MP_ALLOC_ANON)
2921                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2922                                                      NULL);
2923                 }
2924         }
2925         if (ports != NULL) {
2926                 no_link_check = 1;
2927                 RTE_ETH_FOREACH_DEV(pt_id) {
2928                         printf("\nStopping port %d...\n", pt_id);
2929                         fflush(stdout);
2930                         stop_port(pt_id);
2931                 }
2932                 RTE_ETH_FOREACH_DEV(pt_id) {
2933                         printf("\nShutting down port %d...\n", pt_id);
2934                         fflush(stdout);
2935                         close_port(pt_id);
2936                 }
2937         }
2938
2939         if (hot_plug) {
2940                 ret = rte_dev_event_monitor_stop();
2941                 if (ret) {
2942                         RTE_LOG(ERR, EAL,
2943                                 "fail to stop device event monitor.");
2944                         return;
2945                 }
2946
2947                 ret = rte_dev_event_callback_unregister(NULL,
2948                         dev_event_callback, NULL);
2949                 if (ret < 0) {
2950                         RTE_LOG(ERR, EAL,
2951                                 "fail to unregister device event callback.\n");
2952                         return;
2953                 }
2954
2955                 ret = rte_dev_hotplug_handle_disable();
2956                 if (ret) {
2957                         RTE_LOG(ERR, EAL,
2958                                 "fail to disable hotplug handling.\n");
2959                         return;
2960                 }
2961         }
2962         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2963                 if (mempools[i])
2964                         rte_mempool_free(mempools[i]);
2965         }
2966
2967         printf("\nBye...\n");
2968 }
2969
2970 typedef void (*cmd_func_t)(void);
2971 struct pmd_test_command {
2972         const char *cmd_name;
2973         cmd_func_t cmd_func;
2974 };
2975
2976 /* Check the link status of all ports in up to 9s, and print them finally */
2977 static void
2978 check_all_ports_link_status(uint32_t port_mask)
2979 {
2980 #define CHECK_INTERVAL 100 /* 100ms */
2981 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2982         portid_t portid;
2983         uint8_t count, all_ports_up, print_flag = 0;
2984         struct rte_eth_link link;
2985         int ret;
2986
2987         printf("Checking link statuses...\n");
2988         fflush(stdout);
2989         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2990                 all_ports_up = 1;
2991                 RTE_ETH_FOREACH_DEV(portid) {
2992                         if ((port_mask & (1 << portid)) == 0)
2993                                 continue;
2994                         memset(&link, 0, sizeof(link));
2995                         ret = rte_eth_link_get_nowait(portid, &link);
2996                         if (ret < 0) {
2997                                 all_ports_up = 0;
2998                                 if (print_flag == 1)
2999                                         printf("Port %u link get failed: %s\n",
3000                                                 portid, rte_strerror(-ret));
3001                                 continue;
3002                         }
3003                         /* print link status if flag set */
3004                         if (print_flag == 1) {
3005                                 if (link.link_status)
3006                                         printf(
3007                                         "Port%d Link Up. speed %u Mbps- %s\n",
3008                                         portid, link.link_speed,
3009                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3010                                         ("full-duplex") : ("half-duplex\n"));
3011                                 else
3012                                         printf("Port %d Link Down\n", portid);
3013                                 continue;
3014                         }
3015                         /* clear all_ports_up flag if any link down */
3016                         if (link.link_status == ETH_LINK_DOWN) {
3017                                 all_ports_up = 0;
3018                                 break;
3019                         }
3020                 }
3021                 /* after finally printing all link status, get out */
3022                 if (print_flag == 1)
3023                         break;
3024
3025                 if (all_ports_up == 0) {
3026                         fflush(stdout);
3027                         rte_delay_ms(CHECK_INTERVAL);
3028                 }
3029
3030                 /* set the print_flag if all ports up or timeout */
3031                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3032                         print_flag = 1;
3033                 }
3034
3035                 if (lsc_interrupt)
3036                         break;
3037         }
3038 }
3039
3040 /*
3041  * This callback is for remove a port for a device. It has limitation because
3042  * it is not for multiple port removal for a device.
3043  * TODO: the device detach invoke will plan to be removed from user side to
3044  * eal. And convert all PMDs to free port resources on ether device closing.
3045  */
3046 static void
3047 rmv_port_callback(void *arg)
3048 {
3049         int need_to_start = 0;
3050         int org_no_link_check = no_link_check;
3051         portid_t port_id = (intptr_t)arg;
3052         struct rte_device *dev;
3053
3054         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3055
3056         if (!test_done && port_is_forwarding(port_id)) {
3057                 need_to_start = 1;
3058                 stop_packet_forwarding();
3059         }
3060         no_link_check = 1;
3061         stop_port(port_id);
3062         no_link_check = org_no_link_check;
3063
3064         /* Save rte_device pointer before closing ethdev port */
3065         dev = rte_eth_devices[port_id].device;
3066         close_port(port_id);
3067         detach_device(dev); /* might be already removed or have more ports */
3068
3069         if (need_to_start)
3070                 start_packet_forwarding(0);
3071 }
3072
3073 /* This function is used by the interrupt thread */
3074 static int
3075 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3076                   void *ret_param)
3077 {
3078         RTE_SET_USED(param);
3079         RTE_SET_USED(ret_param);
3080
3081         if (type >= RTE_ETH_EVENT_MAX) {
3082                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3083                         port_id, __func__, type);
3084                 fflush(stderr);
3085         } else if (event_print_mask & (UINT32_C(1) << type)) {
3086                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3087                         eth_event_desc[type]);
3088                 fflush(stdout);
3089         }
3090
3091         switch (type) {
3092         case RTE_ETH_EVENT_NEW:
3093                 ports[port_id].need_setup = 1;
3094                 ports[port_id].port_status = RTE_PORT_HANDLING;
3095                 break;
3096         case RTE_ETH_EVENT_INTR_RMV:
3097                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3098                         break;
3099                 if (rte_eal_alarm_set(100000,
3100                                 rmv_port_callback, (void *)(intptr_t)port_id))
3101                         fprintf(stderr, "Could not set up deferred device removal\n");
3102                 break;
3103         default:
3104                 break;
3105         }
3106         return 0;
3107 }
3108
3109 static int
3110 register_eth_event_callback(void)
3111 {
3112         int ret;
3113         enum rte_eth_event_type event;
3114
3115         for (event = RTE_ETH_EVENT_UNKNOWN;
3116                         event < RTE_ETH_EVENT_MAX; event++) {
3117                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3118                                 event,
3119                                 eth_event_callback,
3120                                 NULL);
3121                 if (ret != 0) {
3122                         TESTPMD_LOG(ERR, "Failed to register callback for "
3123                                         "%s event\n", eth_event_desc[event]);
3124                         return -1;
3125                 }
3126         }
3127
3128         return 0;
3129 }
3130
3131 /* This function is used by the interrupt thread */
3132 static void
3133 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3134                              __rte_unused void *arg)
3135 {
3136         uint16_t port_id;
3137         int ret;
3138
3139         if (type >= RTE_DEV_EVENT_MAX) {
3140                 fprintf(stderr, "%s called upon invalid event %d\n",
3141                         __func__, type);
3142                 fflush(stderr);
3143         }
3144
3145         switch (type) {
3146         case RTE_DEV_EVENT_REMOVE:
3147                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3148                         device_name);
3149                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3150                 if (ret) {
3151                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3152                                 device_name);
3153                         return;
3154                 }
3155                 /*
3156                  * Because the user's callback is invoked in eal interrupt
3157                  * callback, the interrupt callback need to be finished before
3158                  * it can be unregistered when detaching device. So finish
3159                  * callback soon and use a deferred removal to detach device
3160                  * is need. It is a workaround, once the device detaching be
3161                  * moved into the eal in the future, the deferred removal could
3162                  * be deleted.
3163                  */
3164                 if (rte_eal_alarm_set(100000,
3165                                 rmv_port_callback, (void *)(intptr_t)port_id))
3166                         RTE_LOG(ERR, EAL,
3167                                 "Could not set up deferred device removal\n");
3168                 break;
3169         case RTE_DEV_EVENT_ADD:
3170                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3171                         device_name);
3172                 /* TODO: After finish kernel driver binding,
3173                  * begin to attach port.
3174                  */
3175                 break;
3176         default:
3177                 break;
3178         }
3179 }
3180
3181 static int
3182 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3183 {
3184         uint16_t i;
3185         int diag;
3186         uint8_t mapping_found = 0;
3187
3188         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3189                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3190                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3191                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3192                                         tx_queue_stats_mappings[i].queue_id,
3193                                         tx_queue_stats_mappings[i].stats_counter_id);
3194                         if (diag != 0)
3195                                 return diag;
3196                         mapping_found = 1;
3197                 }
3198         }
3199         if (mapping_found)
3200                 port->tx_queue_stats_mapping_enabled = 1;
3201         return 0;
3202 }
3203
3204 static int
3205 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3206 {
3207         uint16_t i;
3208         int diag;
3209         uint8_t mapping_found = 0;
3210
3211         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3212                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3213                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3214                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3215                                         rx_queue_stats_mappings[i].queue_id,
3216                                         rx_queue_stats_mappings[i].stats_counter_id);
3217                         if (diag != 0)
3218                                 return diag;
3219                         mapping_found = 1;
3220                 }
3221         }
3222         if (mapping_found)
3223                 port->rx_queue_stats_mapping_enabled = 1;
3224         return 0;
3225 }
3226
3227 static void
3228 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3229 {
3230         int diag = 0;
3231
3232         diag = set_tx_queue_stats_mapping_registers(pi, port);
3233         if (diag != 0) {
3234                 if (diag == -ENOTSUP) {
3235                         port->tx_queue_stats_mapping_enabled = 0;
3236                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3237                 }
3238                 else
3239                         rte_exit(EXIT_FAILURE,
3240                                         "set_tx_queue_stats_mapping_registers "
3241                                         "failed for port id=%d diag=%d\n",
3242                                         pi, diag);
3243         }
3244
3245         diag = set_rx_queue_stats_mapping_registers(pi, port);
3246         if (diag != 0) {
3247                 if (diag == -ENOTSUP) {
3248                         port->rx_queue_stats_mapping_enabled = 0;
3249                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3250                 }
3251                 else
3252                         rte_exit(EXIT_FAILURE,
3253                                         "set_rx_queue_stats_mapping_registers "
3254                                         "failed for port id=%d diag=%d\n",
3255                                         pi, diag);
3256         }
3257 }
3258
3259 static void
3260 rxtx_port_config(struct rte_port *port)
3261 {
3262         uint16_t qid;
3263         uint64_t offloads;
3264
3265         for (qid = 0; qid < nb_rxq; qid++) {
3266                 offloads = port->rx_conf[qid].offloads;
3267                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3268                 if (offloads != 0)
3269                         port->rx_conf[qid].offloads = offloads;
3270
3271                 /* Check if any Rx parameters have been passed */
3272                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3273                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3274
3275                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3276                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3277
3278                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3279                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3280
3281                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3282                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3283
3284                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3285                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3286
3287                 port->nb_rx_desc[qid] = nb_rxd;
3288         }
3289
3290         for (qid = 0; qid < nb_txq; qid++) {
3291                 offloads = port->tx_conf[qid].offloads;
3292                 port->tx_conf[qid] = port->dev_info.default_txconf;
3293                 if (offloads != 0)
3294                         port->tx_conf[qid].offloads = offloads;
3295
3296                 /* Check if any Tx parameters have been passed */
3297                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3298                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3299
3300                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3301                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3302
3303                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3304                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3305
3306                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3307                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3308
3309                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3310                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3311
3312                 port->nb_tx_desc[qid] = nb_txd;
3313         }
3314 }
3315
3316 void
3317 init_port_config(void)
3318 {
3319         portid_t pid;
3320         struct rte_port *port;
3321         int ret;
3322
3323         RTE_ETH_FOREACH_DEV(pid) {
3324                 port = &ports[pid];
3325                 port->dev_conf.fdir_conf = fdir_conf;
3326
3327                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3328                 if (ret != 0)
3329                         return;
3330
3331                 if (nb_rxq > 1) {
3332                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3333                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3334                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3335                 } else {
3336                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3337                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3338                 }
3339
3340                 if (port->dcb_flag == 0) {
3341                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3342                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
3343                         else
3344                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3345                 }
3346
3347                 rxtx_port_config(port);
3348
3349                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3350                 if (ret != 0)
3351                         return;
3352
3353                 map_port_queue_stats_mapping_registers(pid, port);
3354 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3355                 rte_pmd_ixgbe_bypass_init(pid);
3356 #endif
3357
3358                 if (lsc_interrupt &&
3359                     (rte_eth_devices[pid].data->dev_flags &
3360                      RTE_ETH_DEV_INTR_LSC))
3361                         port->dev_conf.intr_conf.lsc = 1;
3362                 if (rmv_interrupt &&
3363                     (rte_eth_devices[pid].data->dev_flags &
3364                      RTE_ETH_DEV_INTR_RMV))
3365                         port->dev_conf.intr_conf.rmv = 1;
3366         }
3367 }
3368
3369 void set_port_slave_flag(portid_t slave_pid)
3370 {
3371         struct rte_port *port;
3372
3373         port = &ports[slave_pid];
3374         port->slave_flag = 1;
3375 }
3376
3377 void clear_port_slave_flag(portid_t slave_pid)
3378 {
3379         struct rte_port *port;
3380
3381         port = &ports[slave_pid];
3382         port->slave_flag = 0;
3383 }
3384
3385 uint8_t port_is_bonding_slave(portid_t slave_pid)
3386 {
3387         struct rte_port *port;
3388
3389         port = &ports[slave_pid];
3390         if ((rte_eth_devices[slave_pid].data->dev_flags &
3391             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3392                 return 1;
3393         return 0;
3394 }
3395
3396 const uint16_t vlan_tags[] = {
3397                 0,  1,  2,  3,  4,  5,  6,  7,
3398                 8,  9, 10, 11,  12, 13, 14, 15,
3399                 16, 17, 18, 19, 20, 21, 22, 23,
3400                 24, 25, 26, 27, 28, 29, 30, 31
3401 };
3402
3403 static  int
3404 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3405                  enum dcb_mode_enable dcb_mode,
3406                  enum rte_eth_nb_tcs num_tcs,
3407                  uint8_t pfc_en)
3408 {
3409         uint8_t i;
3410         int32_t rc;
3411         struct rte_eth_rss_conf rss_conf;
3412
3413         /*
3414          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3415          * given above, and the number of traffic classes available for use.
3416          */
3417         if (dcb_mode == DCB_VT_ENABLED) {
3418                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3419                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3420                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3421                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3422
3423                 /* VMDQ+DCB RX and TX configurations */
3424                 vmdq_rx_conf->enable_default_pool = 0;
3425                 vmdq_rx_conf->default_pool = 0;
3426                 vmdq_rx_conf->nb_queue_pools =
3427                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3428                 vmdq_tx_conf->nb_queue_pools =
3429                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3430
3431                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3432                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3433                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3434                         vmdq_rx_conf->pool_map[i].pools =
3435                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3436                 }
3437                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3438                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3439                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3440                 }
3441
3442                 /* set DCB mode of RX and TX of multiple queues */
3443                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3444                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3445         } else {
3446                 struct rte_eth_dcb_rx_conf *rx_conf =
3447                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3448                 struct rte_eth_dcb_tx_conf *tx_conf =
3449                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3450
3451                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3452                 if (rc != 0)
3453                         return rc;
3454
3455                 rx_conf->nb_tcs = num_tcs;
3456                 tx_conf->nb_tcs = num_tcs;
3457
3458                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3459                         rx_conf->dcb_tc[i] = i % num_tcs;
3460                         tx_conf->dcb_tc[i] = i % num_tcs;
3461                 }
3462
3463                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3464                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3465                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3466         }
3467
3468         if (pfc_en)
3469                 eth_conf->dcb_capability_en =
3470                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3471         else
3472                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3473
3474         return 0;
3475 }
3476
3477 int
3478 init_port_dcb_config(portid_t pid,
3479                      enum dcb_mode_enable dcb_mode,
3480                      enum rte_eth_nb_tcs num_tcs,
3481                      uint8_t pfc_en)
3482 {
3483         struct rte_eth_conf port_conf;
3484         struct rte_port *rte_port;
3485         int retval;
3486         uint16_t i;
3487
3488         rte_port = &ports[pid];
3489
3490         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3491         /* Enter DCB configuration status */
3492         dcb_config = 1;
3493
3494         port_conf.rxmode = rte_port->dev_conf.rxmode;
3495         port_conf.txmode = rte_port->dev_conf.txmode;
3496
3497         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3498         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3499         if (retval < 0)
3500                 return retval;
3501         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3502
3503         /* re-configure the device . */
3504         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3505         if (retval < 0)
3506                 return retval;
3507
3508         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3509         if (retval != 0)
3510                 return retval;
3511
3512         /* If dev_info.vmdq_pool_base is greater than 0,
3513          * the queue id of vmdq pools is started after pf queues.
3514          */
3515         if (dcb_mode == DCB_VT_ENABLED &&
3516             rte_port->dev_info.vmdq_pool_base > 0) {
3517                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3518                         " for port %d.", pid);
3519                 return -1;
3520         }
3521
3522         /* Assume the ports in testpmd have the same dcb capability
3523          * and has the same number of rxq and txq in dcb mode
3524          */
3525         if (dcb_mode == DCB_VT_ENABLED) {
3526                 if (rte_port->dev_info.max_vfs > 0) {
3527                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3528                         nb_txq = rte_port->dev_info.nb_tx_queues;
3529                 } else {
3530                         nb_rxq = rte_port->dev_info.max_rx_queues;
3531                         nb_txq = rte_port->dev_info.max_tx_queues;
3532                 }
3533         } else {
3534                 /*if vt is disabled, use all pf queues */
3535                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3536                         nb_rxq = rte_port->dev_info.max_rx_queues;
3537                         nb_txq = rte_port->dev_info.max_tx_queues;
3538                 } else {
3539                         nb_rxq = (queueid_t)num_tcs;
3540                         nb_txq = (queueid_t)num_tcs;
3541
3542                 }
3543         }
3544         rx_free_thresh = 64;
3545
3546         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3547
3548         rxtx_port_config(rte_port);
3549         /* VLAN filter */
3550         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3551         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3552                 rx_vft_set(pid, vlan_tags[i], 1);
3553
3554         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3555         if (retval != 0)
3556                 return retval;
3557
3558         map_port_queue_stats_mapping_registers(pid, rte_port);
3559
3560         rte_port->dcb_flag = 1;
3561
3562         return 0;
3563 }
3564
3565 static void
3566 init_port(void)
3567 {
3568         /* Configuration of Ethernet ports. */
3569         ports = rte_zmalloc("testpmd: ports",
3570                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3571                             RTE_CACHE_LINE_SIZE);
3572         if (ports == NULL) {
3573                 rte_exit(EXIT_FAILURE,
3574                                 "rte_zmalloc(%d struct rte_port) failed\n",
3575                                 RTE_MAX_ETHPORTS);
3576         }
3577
3578         /* Initialize ports NUMA structures */
3579         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3580         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3581         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3582 }
3583
3584 static void
3585 force_quit(void)
3586 {
3587         pmd_test_exit();
3588         prompt_exit();
3589 }
3590
3591 static void
3592 print_stats(void)
3593 {
3594         uint8_t i;
3595         const char clr[] = { 27, '[', '2', 'J', '\0' };
3596         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3597
3598         /* Clear screen and move to top left */
3599         printf("%s%s", clr, top_left);
3600
3601         printf("\nPort statistics ====================================");
3602         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3603                 nic_stats_display(fwd_ports_ids[i]);
3604
3605         fflush(stdout);
3606 }
3607
3608 static void
3609 signal_handler(int signum)
3610 {
3611         if (signum == SIGINT || signum == SIGTERM) {
3612                 printf("\nSignal %d received, preparing to exit...\n",
3613                                 signum);
3614 #ifdef RTE_LIBRTE_PDUMP
3615                 /* uninitialize packet capture framework */
3616                 rte_pdump_uninit();
3617 #endif
3618 #ifdef RTE_LIBRTE_LATENCY_STATS
3619                 if (latencystats_enabled != 0)
3620                         rte_latencystats_uninit();
3621 #endif
3622                 force_quit();
3623                 /* Set flag to indicate the force termination. */
3624                 f_quit = 1;
3625                 /* exit with the expected status */
3626                 signal(signum, SIG_DFL);
3627                 kill(getpid(), signum);
3628         }
3629 }
3630
3631 int
3632 main(int argc, char** argv)
3633 {
3634         int diag;
3635         portid_t port_id;
3636         uint16_t count;
3637         int ret;
3638
3639         signal(SIGINT, signal_handler);
3640         signal(SIGTERM, signal_handler);
3641
3642         testpmd_logtype = rte_log_register("testpmd");
3643         if (testpmd_logtype < 0)
3644                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3645         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3646
3647         diag = rte_eal_init(argc, argv);
3648         if (diag < 0)
3649                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3650                          rte_strerror(rte_errno));
3651
3652         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3653                 rte_exit(EXIT_FAILURE,
3654                          "Secondary process type not supported.\n");
3655
3656         ret = register_eth_event_callback();
3657         if (ret != 0)
3658                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3659
3660 #ifdef RTE_LIBRTE_PDUMP
3661         /* initialize packet capture framework */
3662         rte_pdump_init();
3663 #endif
3664
3665         count = 0;
3666         RTE_ETH_FOREACH_DEV(port_id) {
3667                 ports_ids[count] = port_id;
3668                 count++;
3669         }
3670         nb_ports = (portid_t) count;
3671         if (nb_ports == 0)
3672                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3673
3674         /* allocate port structures, and init them */
3675         init_port();
3676
3677         set_def_fwd_config();
3678         if (nb_lcores == 0)
3679                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3680                          "Check the core mask argument\n");
3681
3682         /* Bitrate/latency stats disabled by default */
3683 #ifdef RTE_LIBRTE_BITRATE
3684         bitrate_enabled = 0;
3685 #endif
3686 #ifdef RTE_LIBRTE_LATENCY_STATS
3687         latencystats_enabled = 0;
3688 #endif
3689
3690         /* on FreeBSD, mlockall() is disabled by default */
3691 #ifdef RTE_EXEC_ENV_FREEBSD
3692         do_mlockall = 0;
3693 #else
3694         do_mlockall = 1;
3695 #endif
3696
3697         argc -= diag;
3698         argv += diag;
3699         if (argc > 1)
3700                 launch_args_parse(argc, argv);
3701
3702         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3703                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3704                         strerror(errno));
3705         }
3706
3707         if (tx_first && interactive)
3708                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3709                                 "interactive mode.\n");
3710
3711         if (tx_first && lsc_interrupt) {
3712                 printf("Warning: lsc_interrupt needs to be off when "
3713                                 " using tx_first. Disabling.\n");
3714                 lsc_interrupt = 0;
3715         }
3716
3717         if (!nb_rxq && !nb_txq)
3718                 printf("Warning: Either rx or tx queues should be non-zero\n");
3719
3720         if (nb_rxq > 1 && nb_rxq > nb_txq)
3721                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3722                        "but nb_txq=%d will prevent to fully test it.\n",
3723                        nb_rxq, nb_txq);
3724
3725         init_config();
3726
3727         if (hot_plug) {
3728                 ret = rte_dev_hotplug_handle_enable();
3729                 if (ret) {
3730                         RTE_LOG(ERR, EAL,
3731                                 "fail to enable hotplug handling.");
3732                         return -1;
3733                 }
3734
3735                 ret = rte_dev_event_monitor_start();
3736                 if (ret) {
3737                         RTE_LOG(ERR, EAL,
3738                                 "fail to start device event monitoring.");
3739                         return -1;
3740                 }
3741
3742                 ret = rte_dev_event_callback_register(NULL,
3743                         dev_event_callback, NULL);
3744                 if (ret) {
3745                         RTE_LOG(ERR, EAL,
3746                                 "fail  to register device event callback\n");
3747                         return -1;
3748                 }
3749         }
3750
3751         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3752                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3753
3754         /* set all ports to promiscuous mode by default */
3755         RTE_ETH_FOREACH_DEV(port_id) {
3756                 ret = rte_eth_promiscuous_enable(port_id);
3757                 if (ret != 0)
3758                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3759                                 port_id, rte_strerror(-ret));
3760         }
3761
3762         /* Init metrics library */
3763         rte_metrics_init(rte_socket_id());
3764
3765 #ifdef RTE_LIBRTE_LATENCY_STATS
3766         if (latencystats_enabled != 0) {
3767                 int ret = rte_latencystats_init(1, NULL);
3768                 if (ret)
3769                         printf("Warning: latencystats init()"
3770                                 " returned error %d\n", ret);
3771                 printf("Latencystats running on lcore %d\n",
3772                         latencystats_lcore_id);
3773         }
3774 #endif
3775
3776         /* Setup bitrate stats */
3777 #ifdef RTE_LIBRTE_BITRATE
3778         if (bitrate_enabled != 0) {
3779                 bitrate_data = rte_stats_bitrate_create();
3780                 if (bitrate_data == NULL)
3781                         rte_exit(EXIT_FAILURE,
3782                                 "Could not allocate bitrate data.\n");
3783                 rte_stats_bitrate_reg(bitrate_data);
3784         }
3785 #endif
3786
3787 #ifdef RTE_LIBRTE_CMDLINE
3788         if (strlen(cmdline_filename) != 0)
3789                 cmdline_read_from_file(cmdline_filename);
3790
3791         if (interactive == 1) {
3792                 if (auto_start) {
3793                         printf("Start automatic packet forwarding\n");
3794                         start_packet_forwarding(0);
3795                 }
3796                 prompt();
3797                 pmd_test_exit();
3798         } else
3799 #endif
3800         {
3801                 char c;
3802                 int rc;
3803
3804                 f_quit = 0;
3805
3806                 printf("No commandline core given, start packet forwarding\n");
3807                 start_packet_forwarding(tx_first);
3808                 if (stats_period != 0) {
3809                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3810                         uint64_t timer_period;
3811
3812                         /* Convert to number of cycles */
3813                         timer_period = stats_period * rte_get_timer_hz();
3814
3815                         while (f_quit == 0) {
3816                                 cur_time = rte_get_timer_cycles();
3817                                 diff_time += cur_time - prev_time;
3818
3819                                 if (diff_time >= timer_period) {
3820                                         print_stats();
3821                                         /* Reset the timer */
3822                                         diff_time = 0;
3823                                 }
3824                                 /* Sleep to avoid unnecessary checks */
3825                                 prev_time = cur_time;
3826                                 sleep(1);
3827                         }
3828                 }
3829
3830                 printf("Press enter to exit\n");
3831                 rc = read(0, &c, 1);
3832                 pmd_test_exit();
3833                 if (rc < 0)
3834                         return 1;
3835         }
3836
3837         ret = rte_eal_cleanup();
3838         if (ret != 0)
3839                 rte_exit(EXIT_FAILURE,
3840                          "EAL cleanup failed: %s\n", strerror(-ret));
3841
3842         return EXIT_SUCCESS;
3843 }