app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #ifdef RTE_LIBRTE_IEEE1588
 183         &ieee1588_fwd_engine,
 184 #endif
 185         NULL,
 186 };
 187
 188 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 189 uint16_t mempool_flags;
 190
 191 struct fwd_config cur_fwd_config;
 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 193 uint32_t retry_enabled;
 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 196
 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 198 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 199                                       * specified on command-line. */
 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 201
 202 /*
 203  * In container, it cannot terminate the process which running with 'stats-period'
 204  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 205  */
 206 uint8_t f_quit;
 207
 208 /*
 209  * Configuration of packet segments used by the "txonly" processing engine.
 210  */
 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 213         TXONLY_DEF_PACKET_LEN,
 214 };
 215 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 216
 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 218 /**< Split policy for packets to TX. */
 219
 220 uint8_t txonly_multi_flow;
 221 /**< Whether multiple flows are generated in TXONLY mode. */
 222
 223 uint32_t tx_pkt_times_inter;
 224 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 225
 226 uint32_t tx_pkt_times_intra;
 227 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 228
 229 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 230 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 231
 232 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 233 uint8_t dcb_config = 0;
 234
 235 /* Whether the dcb is in testing status */
 236 uint8_t dcb_test = 0;
 237
 238 /*
 239  * Configurable number of RX/TX queues.
 240  */
 241 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 242 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 243 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 244
 245 /*
 246  * Configurable number of RX/TX ring descriptors.
 247  * Defaults are supplied by drivers via ethdev.
 248  */
 249 #define RTE_TEST_RX_DESC_DEFAULT 0
 250 #define RTE_TEST_TX_DESC_DEFAULT 0
 251 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 252 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 253
 254 #define RTE_PMD_PARAM_UNSET -1
 255 /*
 256  * Configurable values of RX and TX ring threshold registers.
 257  */
 258
 259 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 260 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 262
 263 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 264 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 265 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 266
 267 /*
 268  * Configurable value of RX free threshold.
 269  */
 270 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 271
 272 /*
 273  * Configurable value of RX drop enable.
 274  */
 275 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 276
 277 /*
 278  * Configurable value of TX free threshold.
 279  */
 280 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 281
 282 /*
 283  * Configurable value of TX RS bit threshold.
 284  */
 285 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 286
 287 /*
 288  * Configurable value of buffered packets before sending.
 289  */
 290 uint16_t noisy_tx_sw_bufsz;
 291
 292 /*
 293  * Configurable value of packet buffer timeout.
 294  */
 295 uint16_t noisy_tx_sw_buf_flush_time;
 296
 297 /*
 298  * Configurable value for size of VNF internal memory area
 299  * used for simulating noisy neighbour behaviour
 300  */
 301 uint64_t noisy_lkup_mem_sz;
 302
 303 /*
 304  * Configurable value of number of random writes done in
 305  * VNF simulation memory area.
 306  */
 307 uint64_t noisy_lkup_num_writes;
 308
 309 /*
 310  * Configurable value of number of random reads done in
 311  * VNF simulation memory area.
 312  */
 313 uint64_t noisy_lkup_num_reads;
 314
 315 /*
 316  * Configurable value of number of random reads/writes done in
 317  * VNF simulation memory area.
 318  */
 319 uint64_t noisy_lkup_num_reads_writes;
 320
 321 /*
 322  * Receive Side Scaling (RSS) configuration.
 323  */
 324 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 325
 326 /*
 327  * Port topology configuration
 328  */
 329 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 330
 331 /*
 332  * Avoids to flush all the RX streams before starts forwarding.
 333  */
 334 uint8_t no_flush_rx = 0; /* flush by default */
 335
 336 /*
 337  * Flow API isolated mode.
 338  */
 339 uint8_t flow_isolate_all;
 340
 341 /*
 342  * Avoids to check link status when starting/stopping a port.
 343  */
 344 uint8_t no_link_check = 0; /* check by default */
 345
 346 /*
 347  * Don't automatically start all ports in interactive mode.
 348  */
 349 uint8_t no_device_start = 0;
 350
 351 /*
 352  * Enable link status change notification
 353  */
 354 uint8_t lsc_interrupt = 1; /* enabled by default */
 355
 356 /*
 357  * Enable device removal notification.
 358  */
 359 uint8_t rmv_interrupt = 1; /* enabled by default */
 360
 361 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 362
 363 /* After attach, port setup is called on event or by iterator */
 364 bool setup_on_probe_event = true;
 365
 366 /* Clear ptypes on port initialization. */
 367 uint8_t clear_ptypes = true;
 368
 369 /* Pretty printing of ethdev events */
 370 static const char * const eth_event_desc[] = {
 371         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 372         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 373         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 374         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 375         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 376         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 377         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 378         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 379         [RTE_ETH_EVENT_NEW] = "device probed",
 380         [RTE_ETH_EVENT_DESTROY] = "device released",
 381         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 382         [RTE_ETH_EVENT_MAX] = NULL,
 383 };
 384
 385 /*
 386  * Display or mask ether events
 387  * Default to all events except VF_MBOX
 388  */
 389 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 390                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 391                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 392                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 393                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 394                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 395                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 396                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 397 /*
 398  * Decide if all memory are locked for performance.
 399  */
 400 int do_mlockall = 0;
 401
 402 /*
 403  * NIC bypass mode configuration options.
 404  */
 405
 406 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 407 /* The NIC bypass watchdog timeout. */
 408 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 409 #endif
 410
 411
 412 #ifdef RTE_LIBRTE_LATENCY_STATS
 413
 414 /*
 415  * Set when latency stats is enabled in the commandline
 416  */
 417 uint8_t latencystats_enabled;
 418
 419 /*
 420  * Lcore ID to serive latency statistics.
 421  */
 422 lcoreid_t latencystats_lcore_id = -1;
 423
 424 #endif
 425
 426 /*
 427  * Ethernet device configuration.
 428  */
 429 struct rte_eth_rxmode rx_mode = {
 430         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 431                 /**< Default maximum frame length. */
 432 };
 433
 434 struct rte_eth_txmode tx_mode = {
 435         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 436 };
 437
 438 struct rte_fdir_conf fdir_conf = {
 439         .mode = RTE_FDIR_MODE_NONE,
 440         .pballoc = RTE_FDIR_PBALLOC_64K,
 441         .status = RTE_FDIR_REPORT_STATUS,
 442         .mask = {
 443                 .vlan_tci_mask = 0xFFEF,
 444                 .ipv4_mask     = {
 445                         .src_ip = 0xFFFFFFFF,
 446                         .dst_ip = 0xFFFFFFFF,
 447                 },
 448                 .ipv6_mask     = {
 449                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 450                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 451                 },
 452                 .src_port_mask = 0xFFFF,
 453                 .dst_port_mask = 0xFFFF,
 454                 .mac_addr_byte_mask = 0xFF,
 455                 .tunnel_type_mask = 1,
 456                 .tunnel_id_mask = 0xFFFFFFFF,
 457         },
 458         .drop_queue = 127,
 459 };
 460
 461 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 462
 463 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 464 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 465
 466 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 467 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 468
 469 uint16_t nb_tx_queue_stats_mappings = 0;
 470 uint16_t nb_rx_queue_stats_mappings = 0;
 471
 472 /*
 473  * Display zero values by default for xstats
 474  */
 475 uint8_t xstats_hide_zero;
 476
 477 unsigned int num_sockets = 0;
 478 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 479
 480 #ifdef RTE_LIBRTE_BITRATE
 481 /* Bitrate statistics */
 482 struct rte_stats_bitrates *bitrate_data;
 483 lcoreid_t bitrate_lcore_id;
 484 uint8_t bitrate_enabled;
 485 #endif
 486
 487 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 488 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 489
 490 /*
 491  * hexadecimal bitmask of RX mq mode can be enabled.
 492  */
 493 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 494
 495 /* Forward function declarations */
 496 static void setup_attached_port(portid_t pi);
 497 static void map_port_queue_stats_mapping_registers(portid_t pi,
 498                                                    struct rte_port *port);
 499 static void check_all_ports_link_status(uint32_t port_mask);
 500 static int eth_event_callback(portid_t port_id,
 501                               enum rte_eth_event_type type,
 502                               void *param, void *ret_param);
 503 static void dev_event_callback(const char *device_name,
 504                                 enum rte_dev_event_type type,
 505                                 void *param);
 506
 507 /*
 508  * Check if all the ports are started.
 509  * If yes, return positive value. If not, return zero.
 510  */
 511 static int all_ports_started(void);
 512
 513 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 514 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 515
 516 /* Holds the registered mbuf dynamic flags names. */
 517 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 518
 519 /*
 520  * Helper function to check if socket is already discovered.
 521  * If yes, return positive value. If not, return zero.
 522  */
 523 int
 524 new_socket_id(unsigned int socket_id)
 525 {
 526         unsigned int i;
 527
 528         for (i = 0; i < num_sockets; i++) {
 529                 if (socket_ids[i] == socket_id)
 530                         return 0;
 531         }
 532         return 1;
 533 }
 534
 535 /*
 536  * Setup default configuration.
 537  */
 538 static void
 539 set_default_fwd_lcores_config(void)
 540 {
 541         unsigned int i;
 542         unsigned int nb_lc;
 543         unsigned int sock_num;
 544
 545         nb_lc = 0;
 546         for (i = 0; i < RTE_MAX_LCORE; i++) {
 547                 if (!rte_lcore_is_enabled(i))
 548                         continue;
 549                 sock_num = rte_lcore_to_socket_id(i);
 550                 if (new_socket_id(sock_num)) {
 551                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 552                                 rte_exit(EXIT_FAILURE,
 553                                          "Total sockets greater than %u\n",
 554                                          RTE_MAX_NUMA_NODES);
 555                         }
 556                         socket_ids[num_sockets++] = sock_num;
 557                 }
 558                 if (i == rte_get_master_lcore())
 559                         continue;
 560                 fwd_lcores_cpuids[nb_lc++] = i;
 561         }
 562         nb_lcores = (lcoreid_t) nb_lc;
 563         nb_cfg_lcores = nb_lcores;
 564         nb_fwd_lcores = 1;
 565 }
 566
 567 static void
 568 set_def_peer_eth_addrs(void)
 569 {
 570         portid_t i;
 571
 572         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 573                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 574                 peer_eth_addrs[i].addr_bytes[5] = i;
 575         }
 576 }
 577
 578 static void
 579 set_default_fwd_ports_config(void)
 580 {
 581         portid_t pt_id;
 582         int i = 0;
 583
 584         RTE_ETH_FOREACH_DEV(pt_id) {
 585                 fwd_ports_ids[i++] = pt_id;
 586
 587                 /* Update sockets info according to the attached device */
 588                 int socket_id = rte_eth_dev_socket_id(pt_id);
 589                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 590                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 591                                 rte_exit(EXIT_FAILURE,
 592                                          "Total sockets greater than %u\n",
 593                                          RTE_MAX_NUMA_NODES);
 594                         }
 595                         socket_ids[num_sockets++] = socket_id;
 596                 }
 597         }
 598
 599         nb_cfg_ports = nb_ports;
 600         nb_fwd_ports = nb_ports;
 601 }
 602
 603 void
 604 set_def_fwd_config(void)
 605 {
 606         set_default_fwd_lcores_config();
 607         set_def_peer_eth_addrs();
 608         set_default_fwd_ports_config();
 609 }
 610
 611 /* extremely pessimistic estimation of memory required to create a mempool */
 612 static int
 613 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 614 {
 615         unsigned int n_pages, mbuf_per_pg, leftover;
 616         uint64_t total_mem, mbuf_mem, obj_sz;
 617
 618         /* there is no good way to predict how much space the mempool will
 619          * occupy because it will allocate chunks on the fly, and some of those
 620          * will come from default DPDK memory while some will come from our
 621          * external memory, so just assume 128MB will be enough for everyone.
 622          */
 623         uint64_t hdr_mem = 128 << 20;
 624
 625         /* account for possible non-contiguousness */
 626         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 627         if (obj_sz > pgsz) {
 628                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 629                 return -1;
 630         }
 631
 632         mbuf_per_pg = pgsz / obj_sz;
 633         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 634         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 635
 636         mbuf_mem = n_pages * pgsz;
 637
 638         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 639
 640         if (total_mem > SIZE_MAX) {
 641                 TESTPMD_LOG(ERR, "Memory size too big\n");
 642                 return -1;
 643         }
 644         *out = (size_t)total_mem;
 645
 646         return 0;
 647 }
 648
 649 static int
 650 pagesz_flags(uint64_t page_sz)
 651 {
 652         /* as per mmap() manpage, all page sizes are log2 of page size
 653          * shifted by MAP_HUGE_SHIFT
 654          */
 655         int log2 = rte_log2_u64(page_sz);
 656
 657         return (log2 << HUGE_SHIFT);
 658 }
 659
 660 static void *
 661 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 662 {
 663         void *addr;
 664         int flags;
 665
 666         /* allocate anonymous hugepages */
 667         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 668         if (huge)
 669                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 670
 671         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 672         if (addr == MAP_FAILED)
 673                 return NULL;
 674
 675         return addr;
 676 }
 677
 678 struct extmem_param {
 679         void *addr;
 680         size_t len;
 681         size_t pgsz;
 682         rte_iova_t *iova_table;
 683         unsigned int iova_table_len;
 684 };
 685
 686 static int
 687 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 688                 bool huge)
 689 {
 690         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 691                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 692         unsigned int cur_page, n_pages, pgsz_idx;
 693         size_t mem_sz, cur_pgsz;
 694         rte_iova_t *iovas = NULL;
 695         void *addr;
 696         int ret;
 697
 698         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 699                 /* skip anything that is too big */
 700                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 701                         continue;
 702
 703                 cur_pgsz = pgsizes[pgsz_idx];
 704
 705                 /* if we were told not to allocate hugepages, override */
 706                 if (!huge)
 707                         cur_pgsz = sysconf(_SC_PAGESIZE);
 708
 709                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 710                 if (ret < 0) {
 711                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 712                         return -1;
 713                 }
 714
 715                 /* allocate our memory */
 716                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 717
 718                 /* if we couldn't allocate memory with a specified page size,
 719                  * that doesn't mean we can't do it with other page sizes, so
 720                  * try another one.
 721                  */
 722                 if (addr == NULL)
 723                         continue;
 724
 725                 /* store IOVA addresses for every page in this memory area */
 726                 n_pages = mem_sz / cur_pgsz;
 727
 728                 iovas = malloc(sizeof(*iovas) * n_pages);
 729
 730                 if (iovas == NULL) {
 731                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 732                         goto fail;
 733                 }
 734                 /* lock memory if it's not huge pages */
 735                 if (!huge)
 736                         mlock(addr, mem_sz);
 737
 738                 /* populate IOVA addresses */
 739                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 740                         rte_iova_t iova;
 741                         size_t offset;
 742                         void *cur;
 743
 744                         offset = cur_pgsz * cur_page;
 745                         cur = RTE_PTR_ADD(addr, offset);
 746
 747                         /* touch the page before getting its IOVA */
 748                         *(volatile char *)cur = 0;
 749
 750                         iova = rte_mem_virt2iova(cur);
 751
 752                         iovas[cur_page] = iova;
 753                 }
 754
 755                 break;
 756         }
 757         /* if we couldn't allocate anything */
 758         if (iovas == NULL)
 759                 return -1;
 760
 761         param->addr = addr;
 762         param->len = mem_sz;
 763         param->pgsz = cur_pgsz;
 764         param->iova_table = iovas;
 765         param->iova_table_len = n_pages;
 766
 767         return 0;
 768 fail:
 769         if (iovas)
 770                 free(iovas);
 771         if (addr)
 772                 munmap(addr, mem_sz);
 773
 774         return -1;
 775 }
 776
 777 static int
 778 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 779 {
 780         struct extmem_param param;
 781         int socket_id, ret;
 782
 783         memset(&param, 0, sizeof(param));
 784
 785         /* check if our heap exists */
 786         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 787         if (socket_id < 0) {
 788                 /* create our heap */
 789                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 790                 if (ret < 0) {
 791                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 792                         return -1;
 793                 }
 794         }
 795
 796         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 797         if (ret < 0) {
 798                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 799                 return -1;
 800         }
 801
 802         /* we now have a valid memory area, so add it to heap */
 803         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 804                         param.addr, param.len, param.iova_table,
 805                         param.iova_table_len, param.pgsz);
 806
 807         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 808
 809         /* not needed any more */
 810         free(param.iova_table);
 811
 812         if (ret < 0) {
 813                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 814                 munmap(param.addr, param.len);
 815                 return -1;
 816         }
 817
 818         /* success */
 819
 820         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 821                         param.len >> 20);
 822
 823         return 0;
 824 }
 825 static void
 826 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 827              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 828 {
 829         uint16_t pid = 0;
 830         int ret;
 831
 832         RTE_ETH_FOREACH_DEV(pid) {
 833                 struct rte_eth_dev *dev =
 834                         &rte_eth_devices[pid];
 835
 836                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 837                                         memhdr->len);
 838                 if (ret) {
 839                         TESTPMD_LOG(DEBUG,
 840                                     "unable to DMA unmap addr 0x%p "
 841                                     "for device %s\n",
 842                                     memhdr->addr, dev->data->name);
 843                 }
 844         }
 845         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 846         if (ret) {
 847                 TESTPMD_LOG(DEBUG,
 848                             "unable to un-register addr 0x%p\n", memhdr->addr);
 849         }
 850 }
 851
 852 static void
 853 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 854            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 855 {
 856         uint16_t pid = 0;
 857         size_t page_size = sysconf(_SC_PAGESIZE);
 858         int ret;
 859
 860         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 861                                   page_size);
 862         if (ret) {
 863                 TESTPMD_LOG(DEBUG,
 864                             "unable to register addr 0x%p\n", memhdr->addr);
 865                 return;
 866         }
 867         RTE_ETH_FOREACH_DEV(pid) {
 868                 struct rte_eth_dev *dev =
 869                         &rte_eth_devices[pid];
 870
 871                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 872                                       memhdr->len);
 873                 if (ret) {
 874                         TESTPMD_LOG(DEBUG,
 875                                     "unable to DMA map addr 0x%p "
 876                                     "for device %s\n",
 877                                     memhdr->addr, dev->data->name);
 878                 }
 879         }
 880 }
 881
 882 static unsigned int
 883 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 884             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 885 {
 886         struct rte_pktmbuf_extmem *xmem;
 887         unsigned int ext_num, zone_num, elt_num;
 888         uint16_t elt_size;
 889
 890         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 891         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 892         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 893
 894         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 895         if (xmem == NULL) {
 896                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 897                                  "external buffer descriptors\n");
 898                 *ext_mem = NULL;
 899                 return 0;
 900         }
 901         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 902                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 903                 const struct rte_memzone *mz;
 904                 char mz_name[RTE_MEMZONE_NAMESIZE];
 905                 int ret;
 906
 907                 ret = snprintf(mz_name, sizeof(mz_name),
 908                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 909                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 910                         errno = ENAMETOOLONG;
 911                         ext_num = 0;
 912                         break;
 913                 }
 914                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 915                                                  socket_id,
 916                                                  RTE_MEMZONE_IOVA_CONTIG |
 917                                                  RTE_MEMZONE_1GB |
 918                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 919                                                  EXTBUF_ZONE_SIZE);
 920                 if (mz == NULL) {
 921                         /*
 922                          * The caller exits on external buffer creation
 923                          * error, so there is no need to free memzones.
 924                          */
 925                         errno = ENOMEM;
 926                         ext_num = 0;
 927                         break;
 928                 }
 929                 xseg->buf_ptr = mz->addr;
 930                 xseg->buf_iova = mz->iova;
 931                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 932                 xseg->elt_size = elt_size;
 933         }
 934         if (ext_num == 0 && xmem != NULL) {
 935                 free(xmem);
 936                 xmem = NULL;
 937         }
 938         *ext_mem = xmem;
 939         return ext_num;
 940 }
 941
 942 /*
 943  * Configuration initialisation done once at init time.
 944  */
 945 static struct rte_mempool *
 946 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 947                  unsigned int socket_id)
 948 {
 949         char pool_name[RTE_MEMPOOL_NAMESIZE];
 950         struct rte_mempool *rte_mp = NULL;
 951         uint32_t mb_size;
 952
 953         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 954         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 955
 956         TESTPMD_LOG(INFO,
 957                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 958                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 959
 960         switch (mp_alloc_type) {
 961         case MP_ALLOC_NATIVE:
 962                 {
 963                         /* wrapper to rte_mempool_create() */
 964                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 965                                         rte_mbuf_best_mempool_ops());
 966                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 967                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 968                         break;
 969                 }
 970         case MP_ALLOC_ANON:
 971                 {
 972                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 973                                 mb_size, (unsigned int) mb_mempool_cache,
 974                                 sizeof(struct rte_pktmbuf_pool_private),
 975                                 socket_id, mempool_flags);
 976                         if (rte_mp == NULL)
 977                                 goto err;
 978
 979                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 980                                 rte_mempool_free(rte_mp);
 981                                 rte_mp = NULL;
 982                                 goto err;
 983                         }
 984                         rte_pktmbuf_pool_init(rte_mp, NULL);
 985                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 986                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 987                         break;
 988                 }
 989         case MP_ALLOC_XMEM:
 990         case MP_ALLOC_XMEM_HUGE:
 991                 {
 992                         int heap_socket;
 993                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 994
 995                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 996                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 997
 998                         heap_socket =
 999                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1000                         if (heap_socket < 0)
1001                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1002
1003                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1004                                         rte_mbuf_best_mempool_ops());
1005                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1006                                         mb_mempool_cache, 0, mbuf_seg_size,
1007                                         heap_socket);
1008                         break;
1009                 }
1010         case MP_ALLOC_XBUF:
1011                 {
1012                         struct rte_pktmbuf_extmem *ext_mem;
1013                         unsigned int ext_num;
1014
1015                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1016                                                socket_id, pool_name, &ext_mem);
1017                         if (ext_num == 0)
1018                                 rte_exit(EXIT_FAILURE,
1019                                          "Can't create pinned data buffers\n");
1020
1021                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1022                                         rte_mbuf_best_mempool_ops());
1023                         rte_mp = rte_pktmbuf_pool_create_extbuf
1024                                         (pool_name, nb_mbuf, mb_mempool_cache,
1025                                          0, mbuf_seg_size, socket_id,
1026                                          ext_mem, ext_num);
1027                         free(ext_mem);
1028                         break;
1029                 }
1030         default:
1031                 {
1032                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1033                 }
1034         }
1035
1036 err:
1037         if (rte_mp == NULL) {
1038                 rte_exit(EXIT_FAILURE,
1039                         "Creation of mbuf pool for socket %u failed: %s\n",
1040                         socket_id, rte_strerror(rte_errno));
1041         } else if (verbose_level > 0) {
1042                 rte_mempool_dump(stdout, rte_mp);
1043         }
1044         return rte_mp;
1045 }
1046
1047 /*
1048  * Check given socket id is valid or not with NUMA mode,
1049  * if valid, return 0, else return -1
1050  */
1051 static int
1052 check_socket_id(const unsigned int socket_id)
1053 {
1054         static int warning_once = 0;
1055
1056         if (new_socket_id(socket_id)) {
1057                 if (!warning_once && numa_support)
1058                         printf("Warning: NUMA should be configured manually by"
1059                                " using --port-numa-config and"
1060                                " --ring-numa-config parameters along with"
1061                                " --numa.\n");
1062                 warning_once = 1;
1063                 return -1;
1064         }
1065         return 0;
1066 }
1067
1068 /*
1069  * Get the allowed maximum number of RX queues.
1070  * *pid return the port id which has minimal value of
1071  * max_rx_queues in all ports.
1072  */
1073 queueid_t
1074 get_allowed_max_nb_rxq(portid_t *pid)
1075 {
1076         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1077         bool max_rxq_valid = false;
1078         portid_t pi;
1079         struct rte_eth_dev_info dev_info;
1080
1081         RTE_ETH_FOREACH_DEV(pi) {
1082                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1083                         continue;
1084
1085                 max_rxq_valid = true;
1086                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1087                         allowed_max_rxq = dev_info.max_rx_queues;
1088                         *pid = pi;
1089                 }
1090         }
1091         return max_rxq_valid ? allowed_max_rxq : 0;
1092 }
1093
1094 /*
1095  * Check input rxq is valid or not.
1096  * If input rxq is not greater than any of maximum number
1097  * of RX queues of all ports, it is valid.
1098  * if valid, return 0, else return -1
1099  */
1100 int
1101 check_nb_rxq(queueid_t rxq)
1102 {
1103         queueid_t allowed_max_rxq;
1104         portid_t pid = 0;
1105
1106         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1107         if (rxq > allowed_max_rxq) {
1108                 printf("Fail: input rxq (%u) can't be greater "
1109                        "than max_rx_queues (%u) of port %u\n",
1110                        rxq,
1111                        allowed_max_rxq,
1112                        pid);
1113                 return -1;
1114         }
1115         return 0;
1116 }
1117
1118 /*
1119  * Get the allowed maximum number of TX queues.
1120  * *pid return the port id which has minimal value of
1121  * max_tx_queues in all ports.
1122  */
1123 queueid_t
1124 get_allowed_max_nb_txq(portid_t *pid)
1125 {
1126         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1127         bool max_txq_valid = false;
1128         portid_t pi;
1129         struct rte_eth_dev_info dev_info;
1130
1131         RTE_ETH_FOREACH_DEV(pi) {
1132                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1133                         continue;
1134
1135                 max_txq_valid = true;
1136                 if (dev_info.max_tx_queues < allowed_max_txq) {
1137                         allowed_max_txq = dev_info.max_tx_queues;
1138                         *pid = pi;
1139                 }
1140         }
1141         return max_txq_valid ? allowed_max_txq : 0;
1142 }
1143
1144 /*
1145  * Check input txq is valid or not.
1146  * If input txq is not greater than any of maximum number
1147  * of TX queues of all ports, it is valid.
1148  * if valid, return 0, else return -1
1149  */
1150 int
1151 check_nb_txq(queueid_t txq)
1152 {
1153         queueid_t allowed_max_txq;
1154         portid_t pid = 0;
1155
1156         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1157         if (txq > allowed_max_txq) {
1158                 printf("Fail: input txq (%u) can't be greater "
1159                        "than max_tx_queues (%u) of port %u\n",
1160                        txq,
1161                        allowed_max_txq,
1162                        pid);
1163                 return -1;
1164         }
1165         return 0;
1166 }
1167
1168 /*
1169  * Get the allowed maximum number of RXDs of every rx queue.
1170  * *pid return the port id which has minimal value of
1171  * max_rxd in all queues of all ports.
1172  */
1173 static uint16_t
1174 get_allowed_max_nb_rxd(portid_t *pid)
1175 {
1176         uint16_t allowed_max_rxd = UINT16_MAX;
1177         portid_t pi;
1178         struct rte_eth_dev_info dev_info;
1179
1180         RTE_ETH_FOREACH_DEV(pi) {
1181                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1182                         continue;
1183
1184                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1185                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1186                         *pid = pi;
1187                 }
1188         }
1189         return allowed_max_rxd;
1190 }
1191
1192 /*
1193  * Get the allowed minimal number of RXDs of every rx queue.
1194  * *pid return the port id which has minimal value of
1195  * min_rxd in all queues of all ports.
1196  */
1197 static uint16_t
1198 get_allowed_min_nb_rxd(portid_t *pid)
1199 {
1200         uint16_t allowed_min_rxd = 0;
1201         portid_t pi;
1202         struct rte_eth_dev_info dev_info;
1203
1204         RTE_ETH_FOREACH_DEV(pi) {
1205                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1206                         continue;
1207
1208                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1209                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1210                         *pid = pi;
1211                 }
1212         }
1213
1214         return allowed_min_rxd;
1215 }
1216
1217 /*
1218  * Check input rxd is valid or not.
1219  * If input rxd is not greater than any of maximum number
1220  * of RXDs of every Rx queues and is not less than any of
1221  * minimal number of RXDs of every Rx queues, it is valid.
1222  * if valid, return 0, else return -1
1223  */
1224 int
1225 check_nb_rxd(queueid_t rxd)
1226 {
1227         uint16_t allowed_max_rxd;
1228         uint16_t allowed_min_rxd;
1229         portid_t pid = 0;
1230
1231         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1232         if (rxd > allowed_max_rxd) {
1233                 printf("Fail: input rxd (%u) can't be greater "
1234                        "than max_rxds (%u) of port %u\n",
1235                        rxd,
1236                        allowed_max_rxd,
1237                        pid);
1238                 return -1;
1239         }
1240
1241         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1242         if (rxd < allowed_min_rxd) {
1243                 printf("Fail: input rxd (%u) can't be less "
1244                        "than min_rxds (%u) of port %u\n",
1245                        rxd,
1246                        allowed_min_rxd,
1247                        pid);
1248                 return -1;
1249         }
1250
1251         return 0;
1252 }
1253
1254 /*
1255  * Get the allowed maximum number of TXDs of every rx queues.
1256  * *pid return the port id which has minimal value of
1257  * max_txd in every tx queue.
1258  */
1259 static uint16_t
1260 get_allowed_max_nb_txd(portid_t *pid)
1261 {
1262         uint16_t allowed_max_txd = UINT16_MAX;
1263         portid_t pi;
1264         struct rte_eth_dev_info dev_info;
1265
1266         RTE_ETH_FOREACH_DEV(pi) {
1267                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1268                         continue;
1269
1270                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1271                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1272                         *pid = pi;
1273                 }
1274         }
1275         return allowed_max_txd;
1276 }
1277
1278 /*
1279  * Get the allowed maximum number of TXDs of every tx queues.
1280  * *pid return the port id which has minimal value of
1281  * min_txd in every tx queue.
1282  */
1283 static uint16_t
1284 get_allowed_min_nb_txd(portid_t *pid)
1285 {
1286         uint16_t allowed_min_txd = 0;
1287         portid_t pi;
1288         struct rte_eth_dev_info dev_info;
1289
1290         RTE_ETH_FOREACH_DEV(pi) {
1291                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1292                         continue;
1293
1294                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1295                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1296                         *pid = pi;
1297                 }
1298         }
1299
1300         return allowed_min_txd;
1301 }
1302
1303 /*
1304  * Check input txd is valid or not.
1305  * If input txd is not greater than any of maximum number
1306  * of TXDs of every Rx queues, it is valid.
1307  * if valid, return 0, else return -1
1308  */
1309 int
1310 check_nb_txd(queueid_t txd)
1311 {
1312         uint16_t allowed_max_txd;
1313         uint16_t allowed_min_txd;
1314         portid_t pid = 0;
1315
1316         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1317         if (txd > allowed_max_txd) {
1318                 printf("Fail: input txd (%u) can't be greater "
1319                        "than max_txds (%u) of port %u\n",
1320                        txd,
1321                        allowed_max_txd,
1322                        pid);
1323                 return -1;
1324         }
1325
1326         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1327         if (txd < allowed_min_txd) {
1328                 printf("Fail: input txd (%u) can't be less "
1329                        "than min_txds (%u) of port %u\n",
1330                        txd,
1331                        allowed_min_txd,
1332                        pid);
1333                 return -1;
1334         }
1335         return 0;
1336 }
1337
1338
1339 /*
1340  * Get the allowed maximum number of hairpin queues.
1341  * *pid return the port id which has minimal value of
1342  * max_hairpin_queues in all ports.
1343  */
1344 queueid_t
1345 get_allowed_max_nb_hairpinq(portid_t *pid)
1346 {
1347         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1348         portid_t pi;
1349         struct rte_eth_hairpin_cap cap;
1350
1351         RTE_ETH_FOREACH_DEV(pi) {
1352                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1353                         *pid = pi;
1354                         return 0;
1355                 }
1356                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1357                         allowed_max_hairpinq = cap.max_nb_queues;
1358                         *pid = pi;
1359                 }
1360         }
1361         return allowed_max_hairpinq;
1362 }
1363
1364 /*
1365  * Check input hairpin is valid or not.
1366  * If input hairpin is not greater than any of maximum number
1367  * of hairpin queues of all ports, it is valid.
1368  * if valid, return 0, else return -1
1369  */
1370 int
1371 check_nb_hairpinq(queueid_t hairpinq)
1372 {
1373         queueid_t allowed_max_hairpinq;
1374         portid_t pid = 0;
1375
1376         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1377         if (hairpinq > allowed_max_hairpinq) {
1378                 printf("Fail: input hairpin (%u) can't be greater "
1379                        "than max_hairpin_queues (%u) of port %u\n",
1380                        hairpinq, allowed_max_hairpinq, pid);
1381                 return -1;
1382         }
1383         return 0;
1384 }
1385
1386 static void
1387 init_config(void)
1388 {
1389         portid_t pid;
1390         struct rte_port *port;
1391         struct rte_mempool *mbp;
1392         unsigned int nb_mbuf_per_pool;
1393         lcoreid_t  lc_id;
1394         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1395         struct rte_gro_param gro_param;
1396         uint32_t gso_types;
1397         uint16_t data_size;
1398         bool warning = 0;
1399         int k;
1400         int ret;
1401
1402         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1403
1404         /* Configuration of logical cores. */
1405         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1406                                 sizeof(struct fwd_lcore *) * nb_lcores,
1407                                 RTE_CACHE_LINE_SIZE);
1408         if (fwd_lcores == NULL) {
1409                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1410                                                         "failed\n", nb_lcores);
1411         }
1412         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1413                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1414                                                sizeof(struct fwd_lcore),
1415                                                RTE_CACHE_LINE_SIZE);
1416                 if (fwd_lcores[lc_id] == NULL) {
1417                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1418                                                                 "failed\n");
1419                 }
1420                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1421         }
1422
1423         RTE_ETH_FOREACH_DEV(pid) {
1424                 port = &ports[pid];
1425                 /* Apply default TxRx configuration for all ports */
1426                 port->dev_conf.txmode = tx_mode;
1427                 port->dev_conf.rxmode = rx_mode;
1428
1429                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1430                 if (ret != 0)
1431                         rte_exit(EXIT_FAILURE,
1432                                  "rte_eth_dev_info_get() failed\n");
1433
1434                 if (!(port->dev_info.tx_offload_capa &
1435                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1436                         port->dev_conf.txmode.offloads &=
1437                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1438                 if (numa_support) {
1439                         if (port_numa[pid] != NUMA_NO_CONFIG)
1440                                 port_per_socket[port_numa[pid]]++;
1441                         else {
1442                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1443
1444                                 /*
1445                                  * if socket_id is invalid,
1446                                  * set to the first available socket.
1447                                  */
1448                                 if (check_socket_id(socket_id) < 0)
1449                                         socket_id = socket_ids[0];
1450                                 port_per_socket[socket_id]++;
1451                         }
1452                 }
1453
1454                 /* Apply Rx offloads configuration */
1455                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1456                         port->rx_conf[k].offloads =
1457                                 port->dev_conf.rxmode.offloads;
1458                 /* Apply Tx offloads configuration */
1459                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1460                         port->tx_conf[k].offloads =
1461                                 port->dev_conf.txmode.offloads;
1462
1463                 /* set flag to initialize port/queue */
1464                 port->need_reconfig = 1;
1465                 port->need_reconfig_queues = 1;
1466                 port->tx_metadata = 0;
1467
1468                 /* Check for maximum number of segments per MTU. Accordingly
1469                  * update the mbuf data size.
1470                  */
1471                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1472                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1473                         data_size = rx_mode.max_rx_pkt_len /
1474                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1475
1476                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1477                                                         mbuf_data_size) {
1478                                 mbuf_data_size = data_size +
1479                                                  RTE_PKTMBUF_HEADROOM;
1480                                 warning = 1;
1481                         }
1482                 }
1483         }
1484
1485         if (warning)
1486                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1487                             mbuf_data_size);
1488
1489         /*
1490          * Create pools of mbuf.
1491          * If NUMA support is disabled, create a single pool of mbuf in
1492          * socket 0 memory by default.
1493          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1494          *
1495          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1496          * nb_txd can be configured at run time.
1497          */
1498         if (param_total_num_mbufs)
1499                 nb_mbuf_per_pool = param_total_num_mbufs;
1500         else {
1501                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1502                         (nb_lcores * mb_mempool_cache) +
1503                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1504                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1505         }
1506
1507         if (numa_support) {
1508                 uint8_t i;
1509
1510                 for (i = 0; i < num_sockets; i++)
1511                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1512                                                        nb_mbuf_per_pool,
1513                                                        socket_ids[i]);
1514         } else {
1515                 if (socket_num == UMA_NO_CONFIG)
1516                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1517                                                        nb_mbuf_per_pool, 0);
1518                 else
1519                         mempools[socket_num] = mbuf_pool_create
1520                                                         (mbuf_data_size,
1521                                                          nb_mbuf_per_pool,
1522                                                          socket_num);
1523         }
1524
1525         init_port_config();
1526
1527         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1528                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1529         /*
1530          * Records which Mbuf pool to use by each logical core, if needed.
1531          */
1532         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1533                 mbp = mbuf_pool_find(
1534                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1535
1536                 if (mbp == NULL)
1537                         mbp = mbuf_pool_find(0);
1538                 fwd_lcores[lc_id]->mbp = mbp;
1539                 /* initialize GSO context */
1540                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1541                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1542                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1543                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1544                         RTE_ETHER_CRC_LEN;
1545                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1546         }
1547
1548         /* Configuration of packet forwarding streams. */
1549         if (init_fwd_streams() < 0)
1550                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1551
1552         fwd_config_setup();
1553
1554         /* create a gro context for each lcore */
1555         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1556         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1557         gro_param.max_item_per_flow = MAX_PKT_BURST;
1558         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1559                 gro_param.socket_id = rte_lcore_to_socket_id(
1560                                 fwd_lcores_cpuids[lc_id]);
1561                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1562                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1563                         rte_exit(EXIT_FAILURE,
1564                                         "rte_gro_ctx_create() failed\n");
1565                 }
1566         }
1567 }
1568
1569
1570 void
1571 reconfig(portid_t new_port_id, unsigned socket_id)
1572 {
1573         struct rte_port *port;
1574         int ret;
1575
1576         /* Reconfiguration of Ethernet ports. */
1577         port = &ports[new_port_id];
1578
1579         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1580         if (ret != 0)
1581                 return;
1582
1583         /* set flag to initialize port/queue */
1584         port->need_reconfig = 1;
1585         port->need_reconfig_queues = 1;
1586         port->socket_id = socket_id;
1587
1588         init_port_config();
1589 }
1590
1591
1592 int
1593 init_fwd_streams(void)
1594 {
1595         portid_t pid;
1596         struct rte_port *port;
1597         streamid_t sm_id, nb_fwd_streams_new;
1598         queueid_t q;
1599
1600         /* set socket id according to numa or not */
1601         RTE_ETH_FOREACH_DEV(pid) {
1602                 port = &ports[pid];
1603                 if (nb_rxq > port->dev_info.max_rx_queues) {
1604                         printf("Fail: nb_rxq(%d) is greater than "
1605                                 "max_rx_queues(%d)\n", nb_rxq,
1606                                 port->dev_info.max_rx_queues);
1607                         return -1;
1608                 }
1609                 if (nb_txq > port->dev_info.max_tx_queues) {
1610                         printf("Fail: nb_txq(%d) is greater than "
1611                                 "max_tx_queues(%d)\n", nb_txq,
1612                                 port->dev_info.max_tx_queues);
1613                         return -1;
1614                 }
1615                 if (numa_support) {
1616                         if (port_numa[pid] != NUMA_NO_CONFIG)
1617                                 port->socket_id = port_numa[pid];
1618                         else {
1619                                 port->socket_id = rte_eth_dev_socket_id(pid);
1620
1621                                 /*
1622                                  * if socket_id is invalid,
1623                                  * set to the first available socket.
1624                                  */
1625                                 if (check_socket_id(port->socket_id) < 0)
1626                                         port->socket_id = socket_ids[0];
1627                         }
1628                 }
1629                 else {
1630                         if (socket_num == UMA_NO_CONFIG)
1631                                 port->socket_id = 0;
1632                         else
1633                                 port->socket_id = socket_num;
1634                 }
1635         }
1636
1637         q = RTE_MAX(nb_rxq, nb_txq);
1638         if (q == 0) {
1639                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1640                 return -1;
1641         }
1642         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1643         if (nb_fwd_streams_new == nb_fwd_streams)
1644                 return 0;
1645         /* clear the old */
1646         if (fwd_streams != NULL) {
1647                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1648                         if (fwd_streams[sm_id] == NULL)
1649                                 continue;
1650                         rte_free(fwd_streams[sm_id]);
1651                         fwd_streams[sm_id] = NULL;
1652                 }
1653                 rte_free(fwd_streams);
1654                 fwd_streams = NULL;
1655         }
1656
1657         /* init new */
1658         nb_fwd_streams = nb_fwd_streams_new;
1659         if (nb_fwd_streams) {
1660                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1661                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1662                         RTE_CACHE_LINE_SIZE);
1663                 if (fwd_streams == NULL)
1664                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1665                                  " (struct fwd_stream *)) failed\n",
1666                                  nb_fwd_streams);
1667
1668                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1669                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1670                                 " struct fwd_stream", sizeof(struct fwd_stream),
1671                                 RTE_CACHE_LINE_SIZE);
1672                         if (fwd_streams[sm_id] == NULL)
1673                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1674                                          "(struct fwd_stream) failed\n");
1675                 }
1676         }
1677
1678         return 0;
1679 }
1680
1681 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1682 static void
1683 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1684 {
1685         uint64_t total_burst, sburst;
1686         uint64_t nb_burst;
1687         uint64_t burst_stats[4];
1688         uint16_t pktnb_stats[4];
1689         uint16_t nb_pkt;
1690         int burst_percent[4], sburstp;
1691         int i;
1692
1693         /*
1694          * First compute the total number of packet bursts and the
1695          * two highest numbers of bursts of the same number of packets.
1696          */
1697         memset(&burst_stats, 0x0, sizeof(burst_stats));
1698         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1699
1700         /* Show stats for 0 burst size always */
1701         total_burst = pbs->pkt_burst_spread[0];
1702         burst_stats[0] = pbs->pkt_burst_spread[0];
1703         pktnb_stats[0] = 0;
1704
1705         /* Find the next 2 burst sizes with highest occurrences. */
1706         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1707                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1708
1709                 if (nb_burst == 0)
1710                         continue;
1711
1712                 total_burst += nb_burst;
1713
1714                 if (nb_burst > burst_stats[1]) {
1715                         burst_stats[2] = burst_stats[1];
1716                         pktnb_stats[2] = pktnb_stats[1];
1717                         burst_stats[1] = nb_burst;
1718                         pktnb_stats[1] = nb_pkt;
1719                 } else if (nb_burst > burst_stats[2]) {
1720                         burst_stats[2] = nb_burst;
1721                         pktnb_stats[2] = nb_pkt;
1722                 }
1723         }
1724         if (total_burst == 0)
1725                 return;
1726
1727         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1728         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1729                 if (i == 3) {
1730                         printf("%d%% of other]\n", 100 - sburstp);
1731                         return;
1732                 }
1733
1734                 sburst += burst_stats[i];
1735                 if (sburst == total_burst) {
1736                         printf("%d%% of %d pkts]\n",
1737                                 100 - sburstp, (int) pktnb_stats[i]);
1738                         return;
1739                 }
1740
1741                 burst_percent[i] =
1742                         (double)burst_stats[i] / total_burst * 100;
1743                 printf("%d%% of %d pkts + ",
1744                         burst_percent[i], (int) pktnb_stats[i]);
1745                 sburstp += burst_percent[i];
1746         }
1747 }
1748 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1749
1750 static void
1751 fwd_stream_stats_display(streamid_t stream_id)
1752 {
1753         struct fwd_stream *fs;
1754         static const char *fwd_top_stats_border = "-------";
1755
1756         fs = fwd_streams[stream_id];
1757         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1758             (fs->fwd_dropped == 0))
1759                 return;
1760         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1761                "TX Port=%2d/Queue=%2d %s\n",
1762                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1763                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1764         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1765                " TX-dropped: %-14"PRIu64,
1766                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1767
1768         /* if checksum mode */
1769         if (cur_fwd_eng == &csum_fwd_engine) {
1770                 printf("  RX- bad IP checksum: %-14"PRIu64
1771                        "  Rx- bad L4 checksum: %-14"PRIu64
1772                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1773                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1774                         fs->rx_bad_outer_l4_csum);
1775         } else {
1776                 printf("\n");
1777         }
1778
1779 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1780         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1781         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1782 #endif
1783 }
1784
1785 void
1786 fwd_stats_display(void)
1787 {
1788         static const char *fwd_stats_border = "----------------------";
1789         static const char *acc_stats_border = "+++++++++++++++";
1790         struct {
1791                 struct fwd_stream *rx_stream;
1792                 struct fwd_stream *tx_stream;
1793                 uint64_t tx_dropped;
1794                 uint64_t rx_bad_ip_csum;
1795                 uint64_t rx_bad_l4_csum;
1796                 uint64_t rx_bad_outer_l4_csum;
1797         } ports_stats[RTE_MAX_ETHPORTS];
1798         uint64_t total_rx_dropped = 0;
1799         uint64_t total_tx_dropped = 0;
1800         uint64_t total_rx_nombuf = 0;
1801         struct rte_eth_stats stats;
1802 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1803         uint64_t fwd_cycles = 0;
1804 #endif
1805         uint64_t total_recv = 0;
1806         uint64_t total_xmit = 0;
1807         struct rte_port *port;
1808         streamid_t sm_id;
1809         portid_t pt_id;
1810         int i;
1811
1812         memset(ports_stats, 0, sizeof(ports_stats));
1813
1814         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1815                 struct fwd_stream *fs = fwd_streams[sm_id];
1816
1817                 if (cur_fwd_config.nb_fwd_streams >
1818                     cur_fwd_config.nb_fwd_ports) {
1819                         fwd_stream_stats_display(sm_id);
1820                 } else {
1821                         ports_stats[fs->tx_port].tx_stream = fs;
1822                         ports_stats[fs->rx_port].rx_stream = fs;
1823                 }
1824
1825                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1826
1827                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1828                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1829                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1830                                 fs->rx_bad_outer_l4_csum;
1831
1832 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1833                 fwd_cycles += fs->core_cycles;
1834 #endif
1835         }
1836         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1837                 uint8_t j;
1838
1839                 pt_id = fwd_ports_ids[i];
1840                 port = &ports[pt_id];
1841
1842                 rte_eth_stats_get(pt_id, &stats);
1843                 stats.ipackets -= port->stats.ipackets;
1844                 stats.opackets -= port->stats.opackets;
1845                 stats.ibytes -= port->stats.ibytes;
1846                 stats.obytes -= port->stats.obytes;
1847                 stats.imissed -= port->stats.imissed;
1848                 stats.oerrors -= port->stats.oerrors;
1849                 stats.rx_nombuf -= port->stats.rx_nombuf;
1850
1851                 total_recv += stats.ipackets;
1852                 total_xmit += stats.opackets;
1853                 total_rx_dropped += stats.imissed;
1854                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1855                 total_tx_dropped += stats.oerrors;
1856                 total_rx_nombuf  += stats.rx_nombuf;
1857
1858                 printf("\n  %s Forward statistics for port %-2d %s\n",
1859                        fwd_stats_border, pt_id, fwd_stats_border);
1860
1861                 if (!port->rx_queue_stats_mapping_enabled &&
1862                     !port->tx_queue_stats_mapping_enabled) {
1863                         printf("  RX-packets: %-14"PRIu64
1864                                " RX-dropped: %-14"PRIu64
1865                                "RX-total: %-"PRIu64"\n",
1866                                stats.ipackets, stats.imissed,
1867                                stats.ipackets + stats.imissed);
1868
1869                         if (cur_fwd_eng == &csum_fwd_engine)
1870                                 printf("  Bad-ipcsum: %-14"PRIu64
1871                                        " Bad-l4csum: %-14"PRIu64
1872                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1873                                        ports_stats[pt_id].rx_bad_ip_csum,
1874                                        ports_stats[pt_id].rx_bad_l4_csum,
1875                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1876                         if (stats.ierrors + stats.rx_nombuf > 0) {
1877                                 printf("  RX-error: %-"PRIu64"\n",
1878                                        stats.ierrors);
1879                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1880                                        stats.rx_nombuf);
1881                         }
1882
1883                         printf("  TX-packets: %-14"PRIu64
1884                                " TX-dropped: %-14"PRIu64
1885                                "TX-total: %-"PRIu64"\n",
1886                                stats.opackets, ports_stats[pt_id].tx_dropped,
1887                                stats.opackets + ports_stats[pt_id].tx_dropped);
1888                 } else {
1889                         printf("  RX-packets:             %14"PRIu64
1890                                "    RX-dropped:%14"PRIu64
1891                                "    RX-total:%14"PRIu64"\n",
1892                                stats.ipackets, stats.imissed,
1893                                stats.ipackets + stats.imissed);
1894
1895                         if (cur_fwd_eng == &csum_fwd_engine)
1896                                 printf("  Bad-ipcsum:%14"PRIu64
1897                                        "    Bad-l4csum:%14"PRIu64
1898                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1899                                        ports_stats[pt_id].rx_bad_ip_csum,
1900                                        ports_stats[pt_id].rx_bad_l4_csum,
1901                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1902                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1903                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1904                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1905                                        stats.rx_nombuf);
1906                         }
1907
1908                         printf("  TX-packets:             %14"PRIu64
1909                                "    TX-dropped:%14"PRIu64
1910                                "    TX-total:%14"PRIu64"\n",
1911                                stats.opackets, ports_stats[pt_id].tx_dropped,
1912                                stats.opackets + ports_stats[pt_id].tx_dropped);
1913                 }
1914
1915 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1916                 if (ports_stats[pt_id].rx_stream)
1917                         pkt_burst_stats_display("RX",
1918                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1919                 if (ports_stats[pt_id].tx_stream)
1920                         pkt_burst_stats_display("TX",
1921                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1922 #endif
1923
1924                 if (port->rx_queue_stats_mapping_enabled) {
1925                         printf("\n");
1926                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1927                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1928                                        "     RX-errors:%14"PRIu64
1929                                        "    RX-bytes:%14"PRIu64"\n",
1930                                        j, stats.q_ipackets[j],
1931                                        stats.q_errors[j], stats.q_ibytes[j]);
1932                         }
1933                         printf("\n");
1934                 }
1935                 if (port->tx_queue_stats_mapping_enabled) {
1936                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1937                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1938                                        "                                 TX-bytes:%14"
1939                                        PRIu64"\n",
1940                                        j, stats.q_opackets[j],
1941                                        stats.q_obytes[j]);
1942                         }
1943                 }
1944
1945                 printf("  %s--------------------------------%s\n",
1946                        fwd_stats_border, fwd_stats_border);
1947         }
1948
1949         printf("\n  %s Accumulated forward statistics for all ports"
1950                "%s\n",
1951                acc_stats_border, acc_stats_border);
1952         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1953                "%-"PRIu64"\n"
1954                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1955                "%-"PRIu64"\n",
1956                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1957                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1958         if (total_rx_nombuf > 0)
1959                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1960         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1961                "%s\n",
1962                acc_stats_border, acc_stats_border);
1963 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1964 #define CYC_PER_MHZ 1E6
1965         if (total_recv > 0 || total_xmit > 0) {
1966                 uint64_t total_pkts = 0;
1967                 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1968                     strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1969                         total_pkts = total_xmit;
1970                 else
1971                         total_pkts = total_recv;
1972
1973                 printf("\n  CPU cycles/packet=%.2F (total cycles="
1974                        "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1975                        " MHz Clock\n",
1976                        (double) fwd_cycles / total_pkts,
1977                        fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1978                        (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1979         }
1980 #endif
1981 }
1982
1983 void
1984 fwd_stats_reset(void)
1985 {
1986         streamid_t sm_id;
1987         portid_t pt_id;
1988         int i;
1989
1990         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1991                 pt_id = fwd_ports_ids[i];
1992                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1993         }
1994         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1995                 struct fwd_stream *fs = fwd_streams[sm_id];
1996
1997                 fs->rx_packets = 0;
1998                 fs->tx_packets = 0;
1999                 fs->fwd_dropped = 0;
2000                 fs->rx_bad_ip_csum = 0;
2001                 fs->rx_bad_l4_csum = 0;
2002                 fs->rx_bad_outer_l4_csum = 0;
2003
2004 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
2005                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2006                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2007 #endif
2008 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
2009                 fs->core_cycles = 0;
2010 #endif
2011         }
2012 }
2013
2014 static void
2015 flush_fwd_rx_queues(void)
2016 {
2017         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2018         portid_t  rxp;
2019         portid_t port_id;
2020         queueid_t rxq;
2021         uint16_t  nb_rx;
2022         uint16_t  i;
2023         uint8_t   j;
2024         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2025         uint64_t timer_period;
2026
2027         /* convert to number of cycles */
2028         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2029
2030         for (j = 0; j < 2; j++) {
2031                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2032                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2033                                 port_id = fwd_ports_ids[rxp];
2034                                 /**
2035                                 * testpmd can stuck in the below do while loop
2036                                 * if rte_eth_rx_burst() always returns nonzero
2037                                 * packets. So timer is added to exit this loop
2038                                 * after 1sec timer expiry.
2039                                 */
2040                                 prev_tsc = rte_rdtsc();
2041                                 do {
2042                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2043                                                 pkts_burst, MAX_PKT_BURST);
2044                                         for (i = 0; i < nb_rx; i++)
2045                                                 rte_pktmbuf_free(pkts_burst[i]);
2046
2047                                         cur_tsc = rte_rdtsc();
2048                                         diff_tsc = cur_tsc - prev_tsc;
2049                                         timer_tsc += diff_tsc;
2050                                 } while ((nb_rx > 0) &&
2051                                         (timer_tsc < timer_period));
2052                                 timer_tsc = 0;
2053                         }
2054                 }
2055                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2056         }
2057 }
2058
2059 static void
2060 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2061 {
2062         struct fwd_stream **fsm;
2063         streamid_t nb_fs;
2064         streamid_t sm_id;
2065 #ifdef RTE_LIBRTE_BITRATE
2066         uint64_t tics_per_1sec;
2067         uint64_t tics_datum;
2068         uint64_t tics_current;
2069         uint16_t i, cnt_ports;
2070
2071         cnt_ports = nb_ports;
2072         tics_datum = rte_rdtsc();
2073         tics_per_1sec = rte_get_timer_hz();
2074 #endif
2075         fsm = &fwd_streams[fc->stream_idx];
2076         nb_fs = fc->stream_nb;
2077         do {
2078                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2079                         (*pkt_fwd)(fsm[sm_id]);
2080 #ifdef RTE_LIBRTE_BITRATE
2081                 if (bitrate_enabled != 0 &&
2082                                 bitrate_lcore_id == rte_lcore_id()) {
2083                         tics_current = rte_rdtsc();
2084                         if (tics_current - tics_datum >= tics_per_1sec) {
2085                                 /* Periodic bitrate calculation */
2086                                 for (i = 0; i < cnt_ports; i++)
2087                                         rte_stats_bitrate_calc(bitrate_data,
2088                                                 ports_ids[i]);
2089                                 tics_datum = tics_current;
2090                         }
2091                 }
2092 #endif
2093 #ifdef RTE_LIBRTE_LATENCY_STATS
2094                 if (latencystats_enabled != 0 &&
2095                                 latencystats_lcore_id == rte_lcore_id())
2096                         rte_latencystats_update();
2097 #endif
2098
2099         } while (! fc->stopped);
2100 }
2101
2102 static int
2103 start_pkt_forward_on_core(void *fwd_arg)
2104 {
2105         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2106                              cur_fwd_config.fwd_eng->packet_fwd);
2107         return 0;
2108 }
2109
2110 /*
2111  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2112  * Used to start communication flows in network loopback test configurations.
2113  */
2114 static int
2115 run_one_txonly_burst_on_core(void *fwd_arg)
2116 {
2117         struct fwd_lcore *fwd_lc;
2118         struct fwd_lcore tmp_lcore;
2119
2120         fwd_lc = (struct fwd_lcore *) fwd_arg;
2121         tmp_lcore = *fwd_lc;
2122         tmp_lcore.stopped = 1;
2123         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2124         return 0;
2125 }
2126
2127 /*
2128  * Launch packet forwarding:
2129  *     - Setup per-port forwarding context.
2130  *     - launch logical cores with their forwarding configuration.
2131  */
2132 static void
2133 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2134 {
2135         port_fwd_begin_t port_fwd_begin;
2136         unsigned int i;
2137         unsigned int lc_id;
2138         int diag;
2139
2140         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2141         if (port_fwd_begin != NULL) {
2142                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2143                         (*port_fwd_begin)(fwd_ports_ids[i]);
2144         }
2145         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2146                 lc_id = fwd_lcores_cpuids[i];
2147                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2148                         fwd_lcores[i]->stopped = 0;
2149                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2150                                                      fwd_lcores[i], lc_id);
2151                         if (diag != 0)
2152                                 printf("launch lcore %u failed - diag=%d\n",
2153                                        lc_id, diag);
2154                 }
2155         }
2156 }
2157
2158 /*
2159  * Launch packet forwarding configuration.
2160  */
2161 void
2162 start_packet_forwarding(int with_tx_first)
2163 {
2164         port_fwd_begin_t port_fwd_begin;
2165         port_fwd_end_t  port_fwd_end;
2166         struct rte_port *port;
2167         unsigned int i;
2168         portid_t   pt_id;
2169
2170         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2171                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2172
2173         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2174                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2175
2176         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2177                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2178                 (!nb_rxq || !nb_txq))
2179                 rte_exit(EXIT_FAILURE,
2180                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2181                         cur_fwd_eng->fwd_mode_name);
2182
2183         if (all_ports_started() == 0) {
2184                 printf("Not all ports were started\n");
2185                 return;
2186         }
2187         if (test_done == 0) {
2188                 printf("Packet forwarding already started\n");
2189                 return;
2190         }
2191
2192
2193         if(dcb_test) {
2194                 for (i = 0; i < nb_fwd_ports; i++) {
2195                         pt_id = fwd_ports_ids[i];
2196                         port = &ports[pt_id];
2197                         if (!port->dcb_flag) {
2198                                 printf("In DCB mode, all forwarding ports must "
2199                                        "be configured in this mode.\n");
2200                                 return;
2201                         }
2202                 }
2203                 if (nb_fwd_lcores == 1) {
2204                         printf("In DCB mode,the nb forwarding cores "
2205                                "should be larger than 1.\n");
2206                         return;
2207                 }
2208         }
2209         test_done = 0;
2210
2211         fwd_config_setup();
2212
2213         if(!no_flush_rx)
2214                 flush_fwd_rx_queues();
2215
2216         pkt_fwd_config_display(&cur_fwd_config);
2217         rxtx_config_display();
2218
2219         fwd_stats_reset();
2220         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2221                 pt_id = fwd_ports_ids[i];
2222                 port = &ports[pt_id];
2223                 map_port_queue_stats_mapping_registers(pt_id, port);
2224         }
2225         if (with_tx_first) {
2226                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2227                 if (port_fwd_begin != NULL) {
2228                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2229                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2230                 }
2231                 while (with_tx_first--) {
2232                         launch_packet_forwarding(
2233                                         run_one_txonly_burst_on_core);
2234                         rte_eal_mp_wait_lcore();
2235                 }
2236                 port_fwd_end = tx_only_engine.port_fwd_end;
2237                 if (port_fwd_end != NULL) {
2238                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2239                                 (*port_fwd_end)(fwd_ports_ids[i]);
2240                 }
2241         }
2242         launch_packet_forwarding(start_pkt_forward_on_core);
2243 }
2244
2245 void
2246 stop_packet_forwarding(void)
2247 {
2248         port_fwd_end_t port_fwd_end;
2249         lcoreid_t lc_id;
2250         portid_t pt_id;
2251         int i;
2252
2253         if (test_done) {
2254                 printf("Packet forwarding not started\n");
2255                 return;
2256         }
2257         printf("Telling cores to stop...");
2258         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2259                 fwd_lcores[lc_id]->stopped = 1;
2260         printf("\nWaiting for lcores to finish...\n");
2261         rte_eal_mp_wait_lcore();
2262         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2263         if (port_fwd_end != NULL) {
2264                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2265                         pt_id = fwd_ports_ids[i];
2266                         (*port_fwd_end)(pt_id);
2267                 }
2268         }
2269
2270         fwd_stats_display();
2271
2272         printf("\nDone.\n");
2273         test_done = 1;
2274 }
2275
2276 void
2277 dev_set_link_up(portid_t pid)
2278 {
2279         if (rte_eth_dev_set_link_up(pid) < 0)
2280                 printf("\nSet link up fail.\n");
2281 }
2282
2283 void
2284 dev_set_link_down(portid_t pid)
2285 {
2286         if (rte_eth_dev_set_link_down(pid) < 0)
2287                 printf("\nSet link down fail.\n");
2288 }
2289
2290 static int
2291 all_ports_started(void)
2292 {
2293         portid_t pi;
2294         struct rte_port *port;
2295
2296         RTE_ETH_FOREACH_DEV(pi) {
2297                 port = &ports[pi];
2298                 /* Check if there is a port which is not started */
2299                 if ((port->port_status != RTE_PORT_STARTED) &&
2300                         (port->slave_flag == 0))
2301                         return 0;
2302         }
2303
2304         /* No port is not started */
2305         return 1;
2306 }
2307
2308 int
2309 port_is_stopped(portid_t port_id)
2310 {
2311         struct rte_port *port = &ports[port_id];
2312
2313         if ((port->port_status != RTE_PORT_STOPPED) &&
2314             (port->slave_flag == 0))
2315                 return 0;
2316         return 1;
2317 }
2318
2319 int
2320 all_ports_stopped(void)
2321 {
2322         portid_t pi;
2323
2324         RTE_ETH_FOREACH_DEV(pi) {
2325                 if (!port_is_stopped(pi))
2326                         return 0;
2327         }
2328
2329         return 1;
2330 }
2331
2332 int
2333 port_is_started(portid_t port_id)
2334 {
2335         if (port_id_is_invalid(port_id, ENABLED_WARN))
2336                 return 0;
2337
2338         if (ports[port_id].port_status != RTE_PORT_STARTED)
2339                 return 0;
2340
2341         return 1;
2342 }
2343
2344 /* Configure the Rx and Tx hairpin queues for the selected port. */
2345 static int
2346 setup_hairpin_queues(portid_t pi)
2347 {
2348         queueid_t qi;
2349         struct rte_eth_hairpin_conf hairpin_conf = {
2350                 .peer_count = 1,
2351         };
2352         int i;
2353         int diag;
2354         struct rte_port *port = &ports[pi];
2355
2356         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2357                 hairpin_conf.peers[0].port = pi;
2358                 hairpin_conf.peers[0].queue = i + nb_rxq;
2359                 diag = rte_eth_tx_hairpin_queue_setup
2360                         (pi, qi, nb_txd, &hairpin_conf);
2361                 i++;
2362                 if (diag == 0)
2363                         continue;
2364
2365                 /* Fail to setup rx queue, return */
2366                 if (rte_atomic16_cmpset(&(port->port_status),
2367                                         RTE_PORT_HANDLING,
2368                                         RTE_PORT_STOPPED) == 0)
2369                         printf("Port %d can not be set back "
2370                                         "to stopped\n", pi);
2371                 printf("Fail to configure port %d hairpin "
2372                                 "queues\n", pi);
2373                 /* try to reconfigure queues next time */
2374                 port->need_reconfig_queues = 1;
2375                 return -1;
2376         }
2377         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2378                 hairpin_conf.peers[0].port = pi;
2379                 hairpin_conf.peers[0].queue = i + nb_txq;
2380                 diag = rte_eth_rx_hairpin_queue_setup
2381                         (pi, qi, nb_rxd, &hairpin_conf);
2382                 i++;
2383                 if (diag == 0)
2384                         continue;
2385
2386                 /* Fail to setup rx queue, return */
2387                 if (rte_atomic16_cmpset(&(port->port_status),
2388                                         RTE_PORT_HANDLING,
2389                                         RTE_PORT_STOPPED) == 0)
2390                         printf("Port %d can not be set back "
2391                                         "to stopped\n", pi);
2392                 printf("Fail to configure port %d hairpin "
2393                                 "queues\n", pi);
2394                 /* try to reconfigure queues next time */
2395                 port->need_reconfig_queues = 1;
2396                 return -1;
2397         }
2398         return 0;
2399 }
2400
2401 int
2402 start_port(portid_t pid)
2403 {
2404         int diag, need_check_link_status = -1;
2405         portid_t pi;
2406         queueid_t qi;
2407         struct rte_port *port;
2408         struct rte_ether_addr mac_addr;
2409         struct rte_eth_hairpin_cap cap;
2410
2411         if (port_id_is_invalid(pid, ENABLED_WARN))
2412                 return 0;
2413
2414         if(dcb_config)
2415                 dcb_test = 1;
2416         RTE_ETH_FOREACH_DEV(pi) {
2417                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2418                         continue;
2419
2420                 need_check_link_status = 0;
2421                 port = &ports[pi];
2422                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2423                                                  RTE_PORT_HANDLING) == 0) {
2424                         printf("Port %d is now not stopped\n", pi);
2425                         continue;
2426                 }
2427
2428                 if (port->need_reconfig > 0) {
2429                         port->need_reconfig = 0;
2430
2431                         if (flow_isolate_all) {
2432                                 int ret = port_flow_isolate(pi, 1);
2433                                 if (ret) {
2434                                         printf("Failed to apply isolated"
2435                                                " mode on port %d\n", pi);
2436                                         return -1;
2437                                 }
2438                         }
2439                         configure_rxtx_dump_callbacks(0);
2440                         printf("Configuring Port %d (socket %u)\n", pi,
2441                                         port->socket_id);
2442                         if (nb_hairpinq > 0 &&
2443                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2444                                 printf("Port %d doesn't support hairpin "
2445                                        "queues\n", pi);
2446                                 return -1;
2447                         }
2448                         /* configure port */
2449                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2450                                                      nb_txq + nb_hairpinq,
2451                                                      &(port->dev_conf));
2452                         if (diag != 0) {
2453                                 if (rte_atomic16_cmpset(&(port->port_status),
2454                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2455                                         printf("Port %d can not be set back "
2456                                                         "to stopped\n", pi);
2457                                 printf("Fail to configure port %d\n", pi);
2458                                 /* try to reconfigure port next time */
2459                                 port->need_reconfig = 1;
2460                                 return -1;
2461                         }
2462                 }
2463                 if (port->need_reconfig_queues > 0) {
2464                         port->need_reconfig_queues = 0;
2465                         /* setup tx queues */
2466                         for (qi = 0; qi < nb_txq; qi++) {
2467                                 if ((numa_support) &&
2468                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2469                                         diag = rte_eth_tx_queue_setup(pi, qi,
2470                                                 port->nb_tx_desc[qi],
2471                                                 txring_numa[pi],
2472                                                 &(port->tx_conf[qi]));
2473                                 else
2474                                         diag = rte_eth_tx_queue_setup(pi, qi,
2475                                                 port->nb_tx_desc[qi],
2476                                                 port->socket_id,
2477                                                 &(port->tx_conf[qi]));
2478
2479                                 if (diag == 0)
2480                                         continue;
2481
2482                                 /* Fail to setup tx queue, return */
2483                                 if (rte_atomic16_cmpset(&(port->port_status),
2484                                                         RTE_PORT_HANDLING,
2485                                                         RTE_PORT_STOPPED) == 0)
2486                                         printf("Port %d can not be set back "
2487                                                         "to stopped\n", pi);
2488                                 printf("Fail to configure port %d tx queues\n",
2489                                        pi);
2490                                 /* try to reconfigure queues next time */
2491                                 port->need_reconfig_queues = 1;
2492                                 return -1;
2493                         }
2494                         for (qi = 0; qi < nb_rxq; qi++) {
2495                                 /* setup rx queues */
2496                                 if ((numa_support) &&
2497                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2498                                         struct rte_mempool * mp =
2499                                                 mbuf_pool_find(rxring_numa[pi]);
2500                                         if (mp == NULL) {
2501                                                 printf("Failed to setup RX queue:"
2502                                                         "No mempool allocation"
2503                                                         " on the socket %d\n",
2504                                                         rxring_numa[pi]);
2505                                                 return -1;
2506                                         }
2507
2508                                         diag = rte_eth_rx_queue_setup(pi, qi,
2509                                              port->nb_rx_desc[qi],
2510                                              rxring_numa[pi],
2511                                              &(port->rx_conf[qi]),
2512                                              mp);
2513                                 } else {
2514                                         struct rte_mempool *mp =
2515                                                 mbuf_pool_find(port->socket_id);
2516                                         if (mp == NULL) {
2517                                                 printf("Failed to setup RX queue:"
2518                                                         "No mempool allocation"
2519                                                         " on the socket %d\n",
2520                                                         port->socket_id);
2521                                                 return -1;
2522                                         }
2523                                         diag = rte_eth_rx_queue_setup(pi, qi,
2524                                              port->nb_rx_desc[qi],
2525                                              port->socket_id,
2526                                              &(port->rx_conf[qi]),
2527                                              mp);
2528                                 }
2529                                 if (diag == 0)
2530                                         continue;
2531
2532                                 /* Fail to setup rx queue, return */
2533                                 if (rte_atomic16_cmpset(&(port->port_status),
2534                                                         RTE_PORT_HANDLING,
2535                                                         RTE_PORT_STOPPED) == 0)
2536                                         printf("Port %d can not be set back "
2537                                                         "to stopped\n", pi);
2538                                 printf("Fail to configure port %d rx queues\n",
2539                                        pi);
2540                                 /* try to reconfigure queues next time */
2541                                 port->need_reconfig_queues = 1;
2542                                 return -1;
2543                         }
2544                         /* setup hairpin queues */
2545                         if (setup_hairpin_queues(pi) != 0)
2546                                 return -1;
2547                 }
2548                 configure_rxtx_dump_callbacks(verbose_level);
2549                 if (clear_ptypes) {
2550                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2551                                         NULL, 0);
2552                         if (diag < 0)
2553                                 printf(
2554                                 "Port %d: Failed to disable Ptype parsing\n",
2555                                 pi);
2556                 }
2557
2558                 /* start port */
2559                 if (rte_eth_dev_start(pi) < 0) {
2560                         printf("Fail to start port %d\n", pi);
2561
2562                         /* Fail to setup rx queue, return */
2563                         if (rte_atomic16_cmpset(&(port->port_status),
2564                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2565                                 printf("Port %d can not be set back to "
2566                                                         "stopped\n", pi);
2567                         continue;
2568                 }
2569
2570                 if (rte_atomic16_cmpset(&(port->port_status),
2571                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2572                         printf("Port %d can not be set into started\n", pi);
2573
2574                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2575                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2576                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2577                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2578                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2579
2580                 /* at least one port started, need checking link status */
2581                 need_check_link_status = 1;
2582         }
2583
2584         if (need_check_link_status == 1 && !no_link_check)
2585                 check_all_ports_link_status(RTE_PORT_ALL);
2586         else if (need_check_link_status == 0)
2587                 printf("Please stop the ports first\n");
2588
2589         printf("Done\n");
2590         return 0;
2591 }
2592
2593 void
2594 stop_port(portid_t pid)
2595 {
2596         portid_t pi;
2597         struct rte_port *port;
2598         int need_check_link_status = 0;
2599
2600         if (dcb_test) {
2601                 dcb_test = 0;
2602                 dcb_config = 0;
2603         }
2604
2605         if (port_id_is_invalid(pid, ENABLED_WARN))
2606                 return;
2607
2608         printf("Stopping ports...\n");
2609
2610         RTE_ETH_FOREACH_DEV(pi) {
2611                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2612                         continue;
2613
2614                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2615                         printf("Please remove port %d from forwarding configuration.\n", pi);
2616                         continue;
2617                 }
2618
2619                 if (port_is_bonding_slave(pi)) {
2620                         printf("Please remove port %d from bonded device.\n", pi);
2621                         continue;
2622                 }
2623
2624                 port = &ports[pi];
2625                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2626                                                 RTE_PORT_HANDLING) == 0)
2627                         continue;
2628
2629                 rte_eth_dev_stop(pi);
2630
2631                 if (rte_atomic16_cmpset(&(port->port_status),
2632                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2633                         printf("Port %d can not be set into stopped\n", pi);
2634                 need_check_link_status = 1;
2635         }
2636         if (need_check_link_status && !no_link_check)
2637                 check_all_ports_link_status(RTE_PORT_ALL);
2638
2639         printf("Done\n");
2640 }
2641
2642 static void
2643 remove_invalid_ports_in(portid_t *array, portid_t *total)
2644 {
2645         portid_t i;
2646         portid_t new_total = 0;
2647
2648         for (i = 0; i < *total; i++)
2649                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2650                         array[new_total] = array[i];
2651                         new_total++;
2652                 }
2653         *total = new_total;
2654 }
2655
2656 static void
2657 remove_invalid_ports(void)
2658 {
2659         remove_invalid_ports_in(ports_ids, &nb_ports);
2660         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2661         nb_cfg_ports = nb_fwd_ports;
2662 }
2663
2664 void
2665 close_port(portid_t pid)
2666 {
2667         portid_t pi;
2668         struct rte_port *port;
2669
2670         if (port_id_is_invalid(pid, ENABLED_WARN))
2671                 return;
2672
2673         printf("Closing ports...\n");
2674
2675         RTE_ETH_FOREACH_DEV(pi) {
2676                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2677                         continue;
2678
2679                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2680                         printf("Please remove port %d from forwarding configuration.\n", pi);
2681                         continue;
2682                 }
2683
2684                 if (port_is_bonding_slave(pi)) {
2685                         printf("Please remove port %d from bonded device.\n", pi);
2686                         continue;
2687                 }
2688
2689                 port = &ports[pi];
2690                 if (rte_atomic16_cmpset(&(port->port_status),
2691                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2692                         printf("Port %d is already closed\n", pi);
2693                         continue;
2694                 }
2695
2696                 if (rte_atomic16_cmpset(&(port->port_status),
2697                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2698                         printf("Port %d is now not stopped\n", pi);
2699                         continue;
2700                 }
2701
2702                 if (port->flow_list)
2703                         port_flow_flush(pi);
2704                 rte_eth_dev_close(pi);
2705
2706                 remove_invalid_ports();
2707
2708                 if (rte_atomic16_cmpset(&(port->port_status),
2709                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2710                         printf("Port %d cannot be set to closed\n", pi);
2711         }
2712
2713         printf("Done\n");
2714 }
2715
2716 void
2717 reset_port(portid_t pid)
2718 {
2719         int diag;
2720         portid_t pi;
2721         struct rte_port *port;
2722
2723         if (port_id_is_invalid(pid, ENABLED_WARN))
2724                 return;
2725
2726         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2727                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2728                 printf("Can not reset port(s), please stop port(s) first.\n");
2729                 return;
2730         }
2731
2732         printf("Resetting ports...\n");
2733
2734         RTE_ETH_FOREACH_DEV(pi) {
2735                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2736                         continue;
2737
2738                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2739                         printf("Please remove port %d from forwarding "
2740                                "configuration.\n", pi);
2741                         continue;
2742                 }
2743
2744                 if (port_is_bonding_slave(pi)) {
2745                         printf("Please remove port %d from bonded device.\n",
2746                                pi);
2747                         continue;
2748                 }
2749
2750                 diag = rte_eth_dev_reset(pi);
2751                 if (diag == 0) {
2752                         port = &ports[pi];
2753                         port->need_reconfig = 1;
2754                         port->need_reconfig_queues = 1;
2755                 } else {
2756                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2757                 }
2758         }
2759
2760         printf("Done\n");
2761 }
2762
2763 void
2764 attach_port(char *identifier)
2765 {
2766         portid_t pi;
2767         struct rte_dev_iterator iterator;
2768
2769         printf("Attaching a new port...\n");
2770
2771         if (identifier == NULL) {
2772                 printf("Invalid parameters are specified\n");
2773                 return;
2774         }
2775
2776         if (rte_dev_probe(identifier) < 0) {
2777                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2778                 return;
2779         }
2780
2781         /* first attach mode: event */
2782         if (setup_on_probe_event) {
2783                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2784                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2785                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2786                                         ports[pi].need_setup != 0)
2787                                 setup_attached_port(pi);
2788                 return;
2789         }
2790
2791         /* second attach mode: iterator */
2792         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2793                 /* setup ports matching the devargs used for probing */
2794                 if (port_is_forwarding(pi))
2795                         continue; /* port was already attached before */
2796                 setup_attached_port(pi);
2797         }
2798 }
2799
2800 static void
2801 setup_attached_port(portid_t pi)
2802 {
2803         unsigned int socket_id;
2804         int ret;
2805
2806         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2807         /* if socket_id is invalid, set to the first available socket. */
2808         if (check_socket_id(socket_id) < 0)
2809                 socket_id = socket_ids[0];
2810         reconfig(pi, socket_id);
2811         ret = rte_eth_promiscuous_enable(pi);
2812         if (ret != 0)
2813                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2814                         pi, rte_strerror(-ret));
2815
2816         ports_ids[nb_ports++] = pi;
2817         fwd_ports_ids[nb_fwd_ports++] = pi;
2818         nb_cfg_ports = nb_fwd_ports;
2819         ports[pi].need_setup = 0;
2820         ports[pi].port_status = RTE_PORT_STOPPED;
2821
2822         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2823         printf("Done\n");
2824 }
2825
2826 static void
2827 detach_device(struct rte_device *dev)
2828 {
2829         portid_t sibling;
2830
2831         if (dev == NULL) {
2832                 printf("Device already removed\n");
2833                 return;
2834         }
2835
2836         printf("Removing a device...\n");
2837
2838         if (rte_dev_remove(dev) < 0) {
2839                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2840                 return;
2841         }
2842         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2843                 /* reset mapping between old ports and removed device */
2844                 rte_eth_devices[sibling].device = NULL;
2845                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2846                         /* sibling ports are forced to be closed */
2847                         ports[sibling].port_status = RTE_PORT_CLOSED;
2848                         printf("Port %u is closed\n", sibling);
2849                 }
2850         }
2851
2852         remove_invalid_ports();
2853
2854         printf("Device is detached\n");
2855         printf("Now total ports is %d\n", nb_ports);
2856         printf("Done\n");
2857         return;
2858 }
2859
2860 void
2861 detach_port_device(portid_t port_id)
2862 {
2863         if (port_id_is_invalid(port_id, ENABLED_WARN))
2864                 return;
2865
2866         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2867                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2868                         printf("Port not stopped\n");
2869                         return;
2870                 }
2871                 printf("Port was not closed\n");
2872                 if (ports[port_id].flow_list)
2873                         port_flow_flush(port_id);
2874         }
2875
2876         detach_device(rte_eth_devices[port_id].device);
2877 }
2878
2879 void
2880 detach_devargs(char *identifier)
2881 {
2882         struct rte_dev_iterator iterator;
2883         struct rte_devargs da;
2884         portid_t port_id;
2885
2886         printf("Removing a device...\n");
2887
2888         memset(&da, 0, sizeof(da));
2889         if (rte_devargs_parsef(&da, "%s", identifier)) {
2890                 printf("cannot parse identifier\n");
2891                 if (da.args)
2892                         free(da.args);
2893                 return;
2894         }
2895
2896         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2897                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2898                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2899                                 printf("Port %u not stopped\n", port_id);
2900                                 rte_eth_iterator_cleanup(&iterator);
2901                                 return;
2902                         }
2903
2904                         /* sibling ports are forced to be closed */
2905                         if (ports[port_id].flow_list)
2906                                 port_flow_flush(port_id);
2907                         ports[port_id].port_status = RTE_PORT_CLOSED;
2908                         printf("Port %u is now closed\n", port_id);
2909                 }
2910         }
2911
2912         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2913                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2914                             da.name, da.bus->name);
2915                 return;
2916         }
2917
2918         remove_invalid_ports();
2919
2920         printf("Device %s is detached\n", identifier);
2921         printf("Now total ports is %d\n", nb_ports);
2922         printf("Done\n");
2923 }
2924
2925 void
2926 pmd_test_exit(void)
2927 {
2928         portid_t pt_id;
2929         int ret;
2930         int i;
2931
2932         if (test_done == 0)
2933                 stop_packet_forwarding();
2934
2935         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2936                 if (mempools[i]) {
2937                         if (mp_alloc_type == MP_ALLOC_ANON)
2938                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2939                                                      NULL);
2940                 }
2941         }
2942         if (ports != NULL) {
2943                 no_link_check = 1;
2944                 RTE_ETH_FOREACH_DEV(pt_id) {
2945                         printf("\nStopping port %d...\n", pt_id);
2946                         fflush(stdout);
2947                         stop_port(pt_id);
2948                 }
2949                 RTE_ETH_FOREACH_DEV(pt_id) {
2950                         printf("\nShutting down port %d...\n", pt_id);
2951                         fflush(stdout);
2952                         close_port(pt_id);
2953                 }
2954         }
2955
2956         if (hot_plug) {
2957                 ret = rte_dev_event_monitor_stop();
2958                 if (ret) {
2959                         RTE_LOG(ERR, EAL,
2960                                 "fail to stop device event monitor.");
2961                         return;
2962                 }
2963
2964                 ret = rte_dev_event_callback_unregister(NULL,
2965                         dev_event_callback, NULL);
2966                 if (ret < 0) {
2967                         RTE_LOG(ERR, EAL,
2968                                 "fail to unregister device event callback.\n");
2969                         return;
2970                 }
2971
2972                 ret = rte_dev_hotplug_handle_disable();
2973                 if (ret) {
2974                         RTE_LOG(ERR, EAL,
2975                                 "fail to disable hotplug handling.\n");
2976                         return;
2977                 }
2978         }
2979         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2980                 if (mempools[i])
2981                         rte_mempool_free(mempools[i]);
2982         }
2983
2984         printf("\nBye...\n");
2985 }
2986
2987 typedef void (*cmd_func_t)(void);
2988 struct pmd_test_command {
2989         const char *cmd_name;
2990         cmd_func_t cmd_func;
2991 };
2992
2993 /* Check the link status of all ports in up to 9s, and print them finally */
2994 static void
2995 check_all_ports_link_status(uint32_t port_mask)
2996 {
2997 #define CHECK_INTERVAL 100 /* 100ms */
2998 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2999         portid_t portid;
3000         uint8_t count, all_ports_up, print_flag = 0;
3001         struct rte_eth_link link;
3002         int ret;
3003
3004         printf("Checking link statuses...\n");
3005         fflush(stdout);
3006         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3007                 all_ports_up = 1;
3008                 RTE_ETH_FOREACH_DEV(portid) {
3009                         if ((port_mask & (1 << portid)) == 0)
3010                                 continue;
3011                         memset(&link, 0, sizeof(link));
3012                         ret = rte_eth_link_get_nowait(portid, &link);
3013                         if (ret < 0) {
3014                                 all_ports_up = 0;
3015                                 if (print_flag == 1)
3016                                         printf("Port %u link get failed: %s\n",
3017                                                 portid, rte_strerror(-ret));
3018                                 continue;
3019                         }
3020                         /* print link status if flag set */
3021                         if (print_flag == 1) {
3022                                 if (link.link_status)
3023                                         printf(
3024                                         "Port%d Link Up. speed %u Mbps- %s\n",
3025                                         portid, link.link_speed,
3026                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3027                                         ("full-duplex") : ("half-duplex"));
3028                                 else
3029                                         printf("Port %d Link Down\n", portid);
3030                                 continue;
3031                         }
3032                         /* clear all_ports_up flag if any link down */
3033                         if (link.link_status == ETH_LINK_DOWN) {
3034                                 all_ports_up = 0;
3035                                 break;
3036                         }
3037                 }
3038                 /* after finally printing all link status, get out */
3039                 if (print_flag == 1)
3040                         break;
3041
3042                 if (all_ports_up == 0) {
3043                         fflush(stdout);
3044                         rte_delay_ms(CHECK_INTERVAL);
3045                 }
3046
3047                 /* set the print_flag if all ports up or timeout */
3048                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3049                         print_flag = 1;
3050                 }
3051
3052                 if (lsc_interrupt)
3053                         break;
3054         }
3055 }
3056
3057 /*
3058  * This callback is for remove a port for a device. It has limitation because
3059  * it is not for multiple port removal for a device.
3060  * TODO: the device detach invoke will plan to be removed from user side to
3061  * eal. And convert all PMDs to free port resources on ether device closing.
3062  */
3063 static void
3064 rmv_port_callback(void *arg)
3065 {
3066         int need_to_start = 0;
3067         int org_no_link_check = no_link_check;
3068         portid_t port_id = (intptr_t)arg;
3069         struct rte_device *dev;
3070
3071         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3072
3073         if (!test_done && port_is_forwarding(port_id)) {
3074                 need_to_start = 1;
3075                 stop_packet_forwarding();
3076         }
3077         no_link_check = 1;
3078         stop_port(port_id);
3079         no_link_check = org_no_link_check;
3080
3081         /* Save rte_device pointer before closing ethdev port */
3082         dev = rte_eth_devices[port_id].device;
3083         close_port(port_id);
3084         detach_device(dev); /* might be already removed or have more ports */
3085
3086         if (need_to_start)
3087                 start_packet_forwarding(0);
3088 }
3089
3090 /* This function is used by the interrupt thread */
3091 static int
3092 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3093                   void *ret_param)
3094 {
3095         RTE_SET_USED(param);
3096         RTE_SET_USED(ret_param);
3097
3098         if (type >= RTE_ETH_EVENT_MAX) {
3099                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3100                         port_id, __func__, type);
3101                 fflush(stderr);
3102         } else if (event_print_mask & (UINT32_C(1) << type)) {
3103                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3104                         eth_event_desc[type]);
3105                 fflush(stdout);
3106         }
3107
3108         switch (type) {
3109         case RTE_ETH_EVENT_NEW:
3110                 ports[port_id].need_setup = 1;
3111                 ports[port_id].port_status = RTE_PORT_HANDLING;
3112                 break;
3113         case RTE_ETH_EVENT_INTR_RMV:
3114                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3115                         break;
3116                 if (rte_eal_alarm_set(100000,
3117                                 rmv_port_callback, (void *)(intptr_t)port_id))
3118                         fprintf(stderr, "Could not set up deferred device removal\n");
3119                 break;
3120         default:
3121                 break;
3122         }
3123         return 0;
3124 }
3125
3126 static int
3127 register_eth_event_callback(void)
3128 {
3129         int ret;
3130         enum rte_eth_event_type event;
3131
3132         for (event = RTE_ETH_EVENT_UNKNOWN;
3133                         event < RTE_ETH_EVENT_MAX; event++) {
3134                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3135                                 event,
3136                                 eth_event_callback,
3137                                 NULL);
3138                 if (ret != 0) {
3139                         TESTPMD_LOG(ERR, "Failed to register callback for "
3140                                         "%s event\n", eth_event_desc[event]);
3141                         return -1;
3142                 }
3143         }
3144
3145         return 0;
3146 }
3147
3148 /* This function is used by the interrupt thread */
3149 static void
3150 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3151                              __rte_unused void *arg)
3152 {
3153         uint16_t port_id;
3154         int ret;
3155
3156         if (type >= RTE_DEV_EVENT_MAX) {
3157                 fprintf(stderr, "%s called upon invalid event %d\n",
3158                         __func__, type);
3159                 fflush(stderr);
3160         }
3161
3162         switch (type) {
3163         case RTE_DEV_EVENT_REMOVE:
3164                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3165                         device_name);
3166                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3167                 if (ret) {
3168                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3169                                 device_name);
3170                         return;
3171                 }
3172                 /*
3173                  * Because the user's callback is invoked in eal interrupt
3174                  * callback, the interrupt callback need to be finished before
3175                  * it can be unregistered when detaching device. So finish
3176                  * callback soon and use a deferred removal to detach device
3177                  * is need. It is a workaround, once the device detaching be
3178                  * moved into the eal in the future, the deferred removal could
3179                  * be deleted.
3180                  */
3181                 if (rte_eal_alarm_set(100000,
3182                                 rmv_port_callback, (void *)(intptr_t)port_id))
3183                         RTE_LOG(ERR, EAL,
3184                                 "Could not set up deferred device removal\n");
3185                 break;
3186         case RTE_DEV_EVENT_ADD:
3187                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3188                         device_name);
3189                 /* TODO: After finish kernel driver binding,
3190                  * begin to attach port.
3191                  */
3192                 break;
3193         default:
3194                 break;
3195         }
3196 }
3197
3198 static int
3199 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3200 {
3201         uint16_t i;
3202         int diag;
3203         uint8_t mapping_found = 0;
3204
3205         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3206                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3207                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3208                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3209                                         tx_queue_stats_mappings[i].queue_id,
3210                                         tx_queue_stats_mappings[i].stats_counter_id);
3211                         if (diag != 0)
3212                                 return diag;
3213                         mapping_found = 1;
3214                 }
3215         }
3216         if (mapping_found)
3217                 port->tx_queue_stats_mapping_enabled = 1;
3218         return 0;
3219 }
3220
3221 static int
3222 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3223 {
3224         uint16_t i;
3225         int diag;
3226         uint8_t mapping_found = 0;
3227
3228         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3229                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3230                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3231                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3232                                         rx_queue_stats_mappings[i].queue_id,
3233                                         rx_queue_stats_mappings[i].stats_counter_id);
3234                         if (diag != 0)
3235                                 return diag;
3236                         mapping_found = 1;
3237                 }
3238         }
3239         if (mapping_found)
3240                 port->rx_queue_stats_mapping_enabled = 1;
3241         return 0;
3242 }
3243
3244 static void
3245 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3246 {
3247         int diag = 0;
3248
3249         diag = set_tx_queue_stats_mapping_registers(pi, port);
3250         if (diag != 0) {
3251                 if (diag == -ENOTSUP) {
3252                         port->tx_queue_stats_mapping_enabled = 0;
3253                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3254                 }
3255                 else
3256                         rte_exit(EXIT_FAILURE,
3257                                         "set_tx_queue_stats_mapping_registers "
3258                                         "failed for port id=%d diag=%d\n",
3259                                         pi, diag);
3260         }
3261
3262         diag = set_rx_queue_stats_mapping_registers(pi, port);
3263         if (diag != 0) {
3264                 if (diag == -ENOTSUP) {
3265                         port->rx_queue_stats_mapping_enabled = 0;
3266                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3267                 }
3268                 else
3269                         rte_exit(EXIT_FAILURE,
3270                                         "set_rx_queue_stats_mapping_registers "
3271                                         "failed for port id=%d diag=%d\n",
3272                                         pi, diag);
3273         }
3274 }
3275
3276 static void
3277 rxtx_port_config(struct rte_port *port)
3278 {
3279         uint16_t qid;
3280         uint64_t offloads;
3281
3282         for (qid = 0; qid < nb_rxq; qid++) {
3283                 offloads = port->rx_conf[qid].offloads;
3284                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3285                 if (offloads != 0)
3286                         port->rx_conf[qid].offloads = offloads;
3287
3288                 /* Check if any Rx parameters have been passed */
3289                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3290                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3291
3292                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3293                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3294
3295                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3296                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3297
3298                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3299                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3300
3301                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3302                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3303
3304                 port->nb_rx_desc[qid] = nb_rxd;
3305         }
3306
3307         for (qid = 0; qid < nb_txq; qid++) {
3308                 offloads = port->tx_conf[qid].offloads;
3309                 port->tx_conf[qid] = port->dev_info.default_txconf;
3310                 if (offloads != 0)
3311                         port->tx_conf[qid].offloads = offloads;
3312
3313                 /* Check if any Tx parameters have been passed */
3314                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3315                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3316
3317                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3318                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3319
3320                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3321                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3322
3323                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3324                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3325
3326                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3327                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3328
3329                 port->nb_tx_desc[qid] = nb_txd;
3330         }
3331 }
3332
3333 void
3334 init_port_config(void)
3335 {
3336         portid_t pid;
3337         struct rte_port *port;
3338         int ret;
3339
3340         RTE_ETH_FOREACH_DEV(pid) {
3341                 port = &ports[pid];
3342                 port->dev_conf.fdir_conf = fdir_conf;
3343
3344                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3345                 if (ret != 0)
3346                         return;
3347
3348                 if (nb_rxq > 1) {
3349                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3350                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3351                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3352                 } else {
3353                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3354                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3355                 }
3356
3357                 if (port->dcb_flag == 0) {
3358                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3359                                 port->dev_conf.rxmode.mq_mode =
3360                                         (enum rte_eth_rx_mq_mode)
3361                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3362                         else
3363                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3364                 }
3365
3366                 rxtx_port_config(port);
3367
3368                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3369                 if (ret != 0)
3370                         return;
3371
3372                 map_port_queue_stats_mapping_registers(pid, port);
3373 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3374                 rte_pmd_ixgbe_bypass_init(pid);
3375 #endif
3376
3377                 if (lsc_interrupt &&
3378                     (rte_eth_devices[pid].data->dev_flags &
3379                      RTE_ETH_DEV_INTR_LSC))
3380                         port->dev_conf.intr_conf.lsc = 1;
3381                 if (rmv_interrupt &&
3382                     (rte_eth_devices[pid].data->dev_flags &
3383                      RTE_ETH_DEV_INTR_RMV))
3384                         port->dev_conf.intr_conf.rmv = 1;
3385         }
3386 }
3387
3388 void set_port_slave_flag(portid_t slave_pid)
3389 {
3390         struct rte_port *port;
3391
3392         port = &ports[slave_pid];
3393         port->slave_flag = 1;
3394 }
3395
3396 void clear_port_slave_flag(portid_t slave_pid)
3397 {
3398         struct rte_port *port;
3399
3400         port = &ports[slave_pid];
3401         port->slave_flag = 0;
3402 }
3403
3404 uint8_t port_is_bonding_slave(portid_t slave_pid)
3405 {
3406         struct rte_port *port;
3407
3408         port = &ports[slave_pid];
3409         if ((rte_eth_devices[slave_pid].data->dev_flags &
3410             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3411                 return 1;
3412         return 0;
3413 }
3414
3415 const uint16_t vlan_tags[] = {
3416                 0,  1,  2,  3,  4,  5,  6,  7,
3417                 8,  9, 10, 11,  12, 13, 14, 15,
3418                 16, 17, 18, 19, 20, 21, 22, 23,
3419                 24, 25, 26, 27, 28, 29, 30, 31
3420 };
3421
3422 static  int
3423 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3424                  enum dcb_mode_enable dcb_mode,
3425                  enum rte_eth_nb_tcs num_tcs,
3426                  uint8_t pfc_en)
3427 {
3428         uint8_t i;
3429         int32_t rc;
3430         struct rte_eth_rss_conf rss_conf;
3431
3432         /*
3433          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3434          * given above, and the number of traffic classes available for use.
3435          */
3436         if (dcb_mode == DCB_VT_ENABLED) {
3437                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3438                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3439                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3440                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3441
3442                 /* VMDQ+DCB RX and TX configurations */
3443                 vmdq_rx_conf->enable_default_pool = 0;
3444                 vmdq_rx_conf->default_pool = 0;
3445                 vmdq_rx_conf->nb_queue_pools =
3446                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3447                 vmdq_tx_conf->nb_queue_pools =
3448                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3449
3450                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3451                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3452                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3453                         vmdq_rx_conf->pool_map[i].pools =
3454                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3455                 }
3456                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3457                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3458                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3459                 }
3460
3461                 /* set DCB mode of RX and TX of multiple queues */
3462                 eth_conf->rxmode.mq_mode =
3463                                 (enum rte_eth_rx_mq_mode)
3464                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3465                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3466         } else {
3467                 struct rte_eth_dcb_rx_conf *rx_conf =
3468                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3469                 struct rte_eth_dcb_tx_conf *tx_conf =
3470                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3471
3472                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3473
3474                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3475                 if (rc != 0)
3476                         return rc;
3477
3478                 rx_conf->nb_tcs = num_tcs;
3479                 tx_conf->nb_tcs = num_tcs;
3480
3481                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3482                         rx_conf->dcb_tc[i] = i % num_tcs;
3483                         tx_conf->dcb_tc[i] = i % num_tcs;
3484                 }
3485
3486                 eth_conf->rxmode.mq_mode =
3487                                 (enum rte_eth_rx_mq_mode)
3488                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3489                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3490                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3491         }
3492
3493         if (pfc_en)
3494                 eth_conf->dcb_capability_en =
3495                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3496         else
3497                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3498
3499         return 0;
3500 }
3501
3502 int
3503 init_port_dcb_config(portid_t pid,
3504                      enum dcb_mode_enable dcb_mode,
3505                      enum rte_eth_nb_tcs num_tcs,
3506                      uint8_t pfc_en)
3507 {
3508         struct rte_eth_conf port_conf;
3509         struct rte_port *rte_port;
3510         int retval;
3511         uint16_t i;
3512
3513         rte_port = &ports[pid];
3514
3515         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3516         /* Enter DCB configuration status */
3517         dcb_config = 1;
3518
3519         port_conf.rxmode = rte_port->dev_conf.rxmode;
3520         port_conf.txmode = rte_port->dev_conf.txmode;
3521
3522         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3523         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3524         if (retval < 0)
3525                 return retval;
3526         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3527
3528         /* re-configure the device . */
3529         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3530         if (retval < 0)
3531                 return retval;
3532
3533         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3534         if (retval != 0)
3535                 return retval;
3536
3537         /* If dev_info.vmdq_pool_base is greater than 0,
3538          * the queue id of vmdq pools is started after pf queues.
3539          */
3540         if (dcb_mode == DCB_VT_ENABLED &&
3541             rte_port->dev_info.vmdq_pool_base > 0) {
3542                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3543                         " for port %d.", pid);
3544                 return -1;
3545         }
3546
3547         /* Assume the ports in testpmd have the same dcb capability
3548          * and has the same number of rxq and txq in dcb mode
3549          */
3550         if (dcb_mode == DCB_VT_ENABLED) {
3551                 if (rte_port->dev_info.max_vfs > 0) {
3552                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3553                         nb_txq = rte_port->dev_info.nb_tx_queues;
3554                 } else {
3555                         nb_rxq = rte_port->dev_info.max_rx_queues;
3556                         nb_txq = rte_port->dev_info.max_tx_queues;
3557                 }
3558         } else {
3559                 /*if vt is disabled, use all pf queues */
3560                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3561                         nb_rxq = rte_port->dev_info.max_rx_queues;
3562                         nb_txq = rte_port->dev_info.max_tx_queues;
3563                 } else {
3564                         nb_rxq = (queueid_t)num_tcs;
3565                         nb_txq = (queueid_t)num_tcs;
3566
3567                 }
3568         }
3569         rx_free_thresh = 64;
3570
3571         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3572
3573         rxtx_port_config(rte_port);
3574         /* VLAN filter */
3575         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3576         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3577                 rx_vft_set(pid, vlan_tags[i], 1);
3578
3579         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3580         if (retval != 0)
3581                 return retval;
3582
3583         map_port_queue_stats_mapping_registers(pid, rte_port);
3584
3585         rte_port->dcb_flag = 1;
3586
3587         return 0;
3588 }
3589
3590 static void
3591 init_port(void)
3592 {
3593         /* Configuration of Ethernet ports. */
3594         ports = rte_zmalloc("testpmd: ports",
3595                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3596                             RTE_CACHE_LINE_SIZE);
3597         if (ports == NULL) {
3598                 rte_exit(EXIT_FAILURE,
3599                                 "rte_zmalloc(%d struct rte_port) failed\n",
3600                                 RTE_MAX_ETHPORTS);
3601         }
3602
3603         /* Initialize ports NUMA structures */
3604         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3605         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3606         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3607 }
3608
3609 static void
3610 force_quit(void)
3611 {
3612         pmd_test_exit();
3613         prompt_exit();
3614 }
3615
3616 static void
3617 print_stats(void)
3618 {
3619         uint8_t i;
3620         const char clr[] = { 27, '[', '2', 'J', '\0' };
3621         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3622
3623         /* Clear screen and move to top left */
3624         printf("%s%s", clr, top_left);
3625
3626         printf("\nPort statistics ====================================");
3627         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3628                 nic_stats_display(fwd_ports_ids[i]);
3629
3630         fflush(stdout);
3631 }
3632
3633 static void
3634 signal_handler(int signum)
3635 {
3636         if (signum == SIGINT || signum == SIGTERM) {
3637                 printf("\nSignal %d received, preparing to exit...\n",
3638                                 signum);
3639 #ifdef RTE_LIBRTE_PDUMP
3640                 /* uninitialize packet capture framework */
3641                 rte_pdump_uninit();
3642 #endif
3643 #ifdef RTE_LIBRTE_LATENCY_STATS
3644                 if (latencystats_enabled != 0)
3645                         rte_latencystats_uninit();
3646 #endif
3647                 force_quit();
3648                 /* Set flag to indicate the force termination. */
3649                 f_quit = 1;
3650                 /* exit with the expected status */
3651                 signal(signum, SIG_DFL);
3652                 kill(getpid(), signum);
3653         }
3654 }
3655
3656 int
3657 main(int argc, char** argv)
3658 {
3659         int diag;
3660         portid_t port_id;
3661         uint16_t count;
3662         int ret;
3663
3664         signal(SIGINT, signal_handler);
3665         signal(SIGTERM, signal_handler);
3666
3667         testpmd_logtype = rte_log_register("testpmd");
3668         if (testpmd_logtype < 0)
3669                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3670         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3671
3672         diag = rte_eal_init(argc, argv);
3673         if (diag < 0)
3674                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3675                          rte_strerror(rte_errno));
3676
3677         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3678                 rte_exit(EXIT_FAILURE,
3679                          "Secondary process type not supported.\n");
3680
3681         ret = register_eth_event_callback();
3682         if (ret != 0)
3683                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3684
3685 #ifdef RTE_LIBRTE_PDUMP
3686         /* initialize packet capture framework */
3687         rte_pdump_init();
3688 #endif
3689
3690         count = 0;
3691         RTE_ETH_FOREACH_DEV(port_id) {
3692                 ports_ids[count] = port_id;
3693                 count++;
3694         }
3695         nb_ports = (portid_t) count;
3696         if (nb_ports == 0)
3697                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3698
3699         /* allocate port structures, and init them */
3700         init_port();
3701
3702         set_def_fwd_config();
3703         if (nb_lcores == 0)
3704                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3705                          "Check the core mask argument\n");
3706
3707         /* Bitrate/latency stats disabled by default */
3708 #ifdef RTE_LIBRTE_BITRATE
3709         bitrate_enabled = 0;
3710 #endif
3711 #ifdef RTE_LIBRTE_LATENCY_STATS
3712         latencystats_enabled = 0;
3713 #endif
3714
3715         /* on FreeBSD, mlockall() is disabled by default */
3716 #ifdef RTE_EXEC_ENV_FREEBSD
3717         do_mlockall = 0;
3718 #else
3719         do_mlockall = 1;
3720 #endif
3721
3722         argc -= diag;
3723         argv += diag;
3724         if (argc > 1)
3725                 launch_args_parse(argc, argv);
3726
3727         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3728                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3729                         strerror(errno));
3730         }
3731
3732         if (tx_first && interactive)
3733                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3734                                 "interactive mode.\n");
3735
3736         if (tx_first && lsc_interrupt) {
3737                 printf("Warning: lsc_interrupt needs to be off when "
3738                                 " using tx_first. Disabling.\n");
3739                 lsc_interrupt = 0;
3740         }
3741
3742         if (!nb_rxq && !nb_txq)
3743                 printf("Warning: Either rx or tx queues should be non-zero\n");
3744
3745         if (nb_rxq > 1 && nb_rxq > nb_txq)
3746                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3747                        "but nb_txq=%d will prevent to fully test it.\n",
3748                        nb_rxq, nb_txq);
3749
3750         init_config();
3751
3752         if (hot_plug) {
3753                 ret = rte_dev_hotplug_handle_enable();
3754                 if (ret) {
3755                         RTE_LOG(ERR, EAL,
3756                                 "fail to enable hotplug handling.");
3757                         return -1;
3758                 }
3759
3760                 ret = rte_dev_event_monitor_start();
3761                 if (ret) {
3762                         RTE_LOG(ERR, EAL,
3763                                 "fail to start device event monitoring.");
3764                         return -1;
3765                 }
3766
3767                 ret = rte_dev_event_callback_register(NULL,
3768                         dev_event_callback, NULL);
3769                 if (ret) {
3770                         RTE_LOG(ERR, EAL,
3771                                 "fail  to register device event callback\n");
3772                         return -1;
3773                 }
3774         }
3775
3776         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3777                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3778
3779         /* set all ports to promiscuous mode by default */
3780         RTE_ETH_FOREACH_DEV(port_id) {
3781                 ret = rte_eth_promiscuous_enable(port_id);
3782                 if (ret != 0)
3783                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3784                                 port_id, rte_strerror(-ret));
3785         }
3786
3787         /* Init metrics library */
3788         rte_metrics_init(rte_socket_id());
3789
3790 #ifdef RTE_LIBRTE_LATENCY_STATS
3791         if (latencystats_enabled != 0) {
3792                 int ret = rte_latencystats_init(1, NULL);
3793                 if (ret)
3794                         printf("Warning: latencystats init()"
3795                                 " returned error %d\n", ret);
3796                 printf("Latencystats running on lcore %d\n",
3797                         latencystats_lcore_id);
3798         }
3799 #endif
3800
3801         /* Setup bitrate stats */
3802 #ifdef RTE_LIBRTE_BITRATE
3803         if (bitrate_enabled != 0) {
3804                 bitrate_data = rte_stats_bitrate_create();
3805                 if (bitrate_data == NULL)
3806                         rte_exit(EXIT_FAILURE,
3807                                 "Could not allocate bitrate data.\n");
3808                 rte_stats_bitrate_reg(bitrate_data);
3809         }
3810 #endif
3811
3812 #ifdef RTE_LIBRTE_CMDLINE
3813         if (strlen(cmdline_filename) != 0)
3814                 cmdline_read_from_file(cmdline_filename);
3815
3816         if (interactive == 1) {
3817                 if (auto_start) {
3818                         printf("Start automatic packet forwarding\n");
3819                         start_packet_forwarding(0);
3820                 }
3821                 prompt();
3822                 pmd_test_exit();
3823         } else
3824 #endif
3825         {
3826                 char c;
3827                 int rc;
3828
3829                 f_quit = 0;
3830
3831                 printf("No commandline core given, start packet forwarding\n");
3832                 start_packet_forwarding(tx_first);
3833                 if (stats_period != 0) {
3834                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3835                         uint64_t timer_period;
3836
3837                         /* Convert to number of cycles */
3838                         timer_period = stats_period * rte_get_timer_hz();
3839
3840                         while (f_quit == 0) {
3841                                 cur_time = rte_get_timer_cycles();
3842                                 diff_time += cur_time - prev_time;
3843
3844                                 if (diff_time >= timer_period) {
3845                                         print_stats();
3846                                         /* Reset the timer */
3847                                         diff_time = 0;
3848                                 }
3849                                 /* Sleep to avoid unnecessary checks */
3850                                 prev_time = cur_time;
3851                                 sleep(1);
3852                         }
3853                 }
3854
3855                 printf("Press enter to exit\n");
3856                 rc = read(0, &c, 1);
3857                 pmd_test_exit();
3858                 if (rc < 0)
3859                         return 1;
3860         }
3861
3862         ret = rte_eal_cleanup();
3863         if (ret != 0)
3864                 rte_exit(EXIT_FAILURE,
3865                          "EAL cleanup failed: %s\n", strerror(-ret));
3866
3867         return EXIT_SUCCESS;
3868 }