app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 199 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 200                                       * specified on command-line. */
 201 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 202
 203 /*
 204  * In container, it cannot terminate the process which running with 'stats-period'
 205  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 206  */
 207 uint8_t f_quit;
 208
 209 /*
 210  * Configuration of packet segments used by the "txonly" processing engine.
 211  */
 212 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 213 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 214         TXONLY_DEF_PACKET_LEN,
 215 };
 216 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 217
 218 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 219 /**< Split policy for packets to TX. */
 220
 221 uint8_t txonly_multi_flow;
 222 /**< Whether multiple flows are generated in TXONLY mode. */
 223
 224 uint32_t tx_pkt_times_inter;
 225 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 226
 227 uint32_t tx_pkt_times_intra;
 228 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 229
 230 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 231 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 232
 233 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 234 uint8_t dcb_config = 0;
 235
 236 /* Whether the dcb is in testing status */
 237 uint8_t dcb_test = 0;
 238
 239 /*
 240  * Configurable number of RX/TX queues.
 241  */
 242 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 243 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 244 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 245
 246 /*
 247  * Configurable number of RX/TX ring descriptors.
 248  * Defaults are supplied by drivers via ethdev.
 249  */
 250 #define RTE_TEST_RX_DESC_DEFAULT 0
 251 #define RTE_TEST_TX_DESC_DEFAULT 0
 252 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 253 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 254
 255 #define RTE_PMD_PARAM_UNSET -1
 256 /*
 257  * Configurable values of RX and TX ring threshold registers.
 258  */
 259
 260 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 262 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 263
 264 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 265 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 266 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 267
 268 /*
 269  * Configurable value of RX free threshold.
 270  */
 271 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 272
 273 /*
 274  * Configurable value of RX drop enable.
 275  */
 276 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of TX free threshold.
 280  */
 281 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of TX RS bit threshold.
 285  */
 286 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of buffered packets before sending.
 290  */
 291 uint16_t noisy_tx_sw_bufsz;
 292
 293 /*
 294  * Configurable value of packet buffer timeout.
 295  */
 296 uint16_t noisy_tx_sw_buf_flush_time;
 297
 298 /*
 299  * Configurable value for size of VNF internal memory area
 300  * used for simulating noisy neighbour behaviour
 301  */
 302 uint64_t noisy_lkup_mem_sz;
 303
 304 /*
 305  * Configurable value of number of random writes done in
 306  * VNF simulation memory area.
 307  */
 308 uint64_t noisy_lkup_num_writes;
 309
 310 /*
 311  * Configurable value of number of random reads done in
 312  * VNF simulation memory area.
 313  */
 314 uint64_t noisy_lkup_num_reads;
 315
 316 /*
 317  * Configurable value of number of random reads/writes done in
 318  * VNF simulation memory area.
 319  */
 320 uint64_t noisy_lkup_num_reads_writes;
 321
 322 /*
 323  * Receive Side Scaling (RSS) configuration.
 324  */
 325 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 326
 327 /*
 328  * Port topology configuration
 329  */
 330 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 331
 332 /*
 333  * Avoids to flush all the RX streams before starts forwarding.
 334  */
 335 uint8_t no_flush_rx = 0; /* flush by default */
 336
 337 /*
 338  * Flow API isolated mode.
 339  */
 340 uint8_t flow_isolate_all;
 341
 342 /*
 343  * Avoids to check link status when starting/stopping a port.
 344  */
 345 uint8_t no_link_check = 0; /* check by default */
 346
 347 /*
 348  * Don't automatically start all ports in interactive mode.
 349  */
 350 uint8_t no_device_start = 0;
 351
 352 /*
 353  * Enable link status change notification
 354  */
 355 uint8_t lsc_interrupt = 1; /* enabled by default */
 356
 357 /*
 358  * Enable device removal notification.
 359  */
 360 uint8_t rmv_interrupt = 1; /* enabled by default */
 361
 362 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 363
 364 /* After attach, port setup is called on event or by iterator */
 365 bool setup_on_probe_event = true;
 366
 367 /* Clear ptypes on port initialization. */
 368 uint8_t clear_ptypes = true;
 369
 370 /* Pretty printing of ethdev events */
 371 static const char * const eth_event_desc[] = {
 372         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 373         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 374         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 375         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 376         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 377         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 378         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 379         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 380         [RTE_ETH_EVENT_NEW] = "device probed",
 381         [RTE_ETH_EVENT_DESTROY] = "device released",
 382         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 383         [RTE_ETH_EVENT_MAX] = NULL,
 384 };
 385
 386 /*
 387  * Display or mask ether events
 388  * Default to all events except VF_MBOX
 389  */
 390 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 391                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 392                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 393                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 394                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 395                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 396                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 397                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 398 /*
 399  * Decide if all memory are locked for performance.
 400  */
 401 int do_mlockall = 0;
 402
 403 /*
 404  * NIC bypass mode configuration options.
 405  */
 406
 407 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 408 /* The NIC bypass watchdog timeout. */
 409 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 410 #endif
 411
 412
 413 #ifdef RTE_LIBRTE_LATENCY_STATS
 414
 415 /*
 416  * Set when latency stats is enabled in the commandline
 417  */
 418 uint8_t latencystats_enabled;
 419
 420 /*
 421  * Lcore ID to serive latency statistics.
 422  */
 423 lcoreid_t latencystats_lcore_id = -1;
 424
 425 #endif
 426
 427 /*
 428  * Ethernet device configuration.
 429  */
 430 struct rte_eth_rxmode rx_mode = {
 431         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 432                 /**< Default maximum frame length. */
 433 };
 434
 435 struct rte_eth_txmode tx_mode = {
 436         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 437 };
 438
 439 struct rte_fdir_conf fdir_conf = {
 440         .mode = RTE_FDIR_MODE_NONE,
 441         .pballoc = RTE_FDIR_PBALLOC_64K,
 442         .status = RTE_FDIR_REPORT_STATUS,
 443         .mask = {
 444                 .vlan_tci_mask = 0xFFEF,
 445                 .ipv4_mask     = {
 446                         .src_ip = 0xFFFFFFFF,
 447                         .dst_ip = 0xFFFFFFFF,
 448                 },
 449                 .ipv6_mask     = {
 450                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 451                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 452                 },
 453                 .src_port_mask = 0xFFFF,
 454                 .dst_port_mask = 0xFFFF,
 455                 .mac_addr_byte_mask = 0xFF,
 456                 .tunnel_type_mask = 1,
 457                 .tunnel_id_mask = 0xFFFFFFFF,
 458         },
 459         .drop_queue = 127,
 460 };
 461
 462 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 463
 464 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 465 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 466
 467 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 468 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 469
 470 uint16_t nb_tx_queue_stats_mappings = 0;
 471 uint16_t nb_rx_queue_stats_mappings = 0;
 472
 473 /*
 474  * Display zero values by default for xstats
 475  */
 476 uint8_t xstats_hide_zero;
 477
 478 unsigned int num_sockets = 0;
 479 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 480
 481 #ifdef RTE_LIBRTE_BITRATE
 482 /* Bitrate statistics */
 483 struct rte_stats_bitrates *bitrate_data;
 484 lcoreid_t bitrate_lcore_id;
 485 uint8_t bitrate_enabled;
 486 #endif
 487
 488 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 489 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 490
 491 /*
 492  * hexadecimal bitmask of RX mq mode can be enabled.
 493  */
 494 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 495
 496 /* Forward function declarations */
 497 static void setup_attached_port(portid_t pi);
 498 static void map_port_queue_stats_mapping_registers(portid_t pi,
 499                                                    struct rte_port *port);
 500 static void check_all_ports_link_status(uint32_t port_mask);
 501 static int eth_event_callback(portid_t port_id,
 502                               enum rte_eth_event_type type,
 503                               void *param, void *ret_param);
 504 static void dev_event_callback(const char *device_name,
 505                                 enum rte_dev_event_type type,
 506                                 void *param);
 507
 508 /*
 509  * Check if all the ports are started.
 510  * If yes, return positive value. If not, return zero.
 511  */
 512 static int all_ports_started(void);
 513
 514 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 515 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 516
 517 /* Holds the registered mbuf dynamic flags names. */
 518 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 519
 520 /*
 521  * Helper function to check if socket is already discovered.
 522  * If yes, return positive value. If not, return zero.
 523  */
 524 int
 525 new_socket_id(unsigned int socket_id)
 526 {
 527         unsigned int i;
 528
 529         for (i = 0; i < num_sockets; i++) {
 530                 if (socket_ids[i] == socket_id)
 531                         return 0;
 532         }
 533         return 1;
 534 }
 535
 536 /*
 537  * Setup default configuration.
 538  */
 539 static void
 540 set_default_fwd_lcores_config(void)
 541 {
 542         unsigned int i;
 543         unsigned int nb_lc;
 544         unsigned int sock_num;
 545
 546         nb_lc = 0;
 547         for (i = 0; i < RTE_MAX_LCORE; i++) {
 548                 if (!rte_lcore_is_enabled(i))
 549                         continue;
 550                 sock_num = rte_lcore_to_socket_id(i);
 551                 if (new_socket_id(sock_num)) {
 552                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 553                                 rte_exit(EXIT_FAILURE,
 554                                          "Total sockets greater than %u\n",
 555                                          RTE_MAX_NUMA_NODES);
 556                         }
 557                         socket_ids[num_sockets++] = sock_num;
 558                 }
 559                 if (i == rte_get_master_lcore())
 560                         continue;
 561                 fwd_lcores_cpuids[nb_lc++] = i;
 562         }
 563         nb_lcores = (lcoreid_t) nb_lc;
 564         nb_cfg_lcores = nb_lcores;
 565         nb_fwd_lcores = 1;
 566 }
 567
 568 static void
 569 set_def_peer_eth_addrs(void)
 570 {
 571         portid_t i;
 572
 573         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 574                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 575                 peer_eth_addrs[i].addr_bytes[5] = i;
 576         }
 577 }
 578
 579 static void
 580 set_default_fwd_ports_config(void)
 581 {
 582         portid_t pt_id;
 583         int i = 0;
 584
 585         RTE_ETH_FOREACH_DEV(pt_id) {
 586                 fwd_ports_ids[i++] = pt_id;
 587
 588                 /* Update sockets info according to the attached device */
 589                 int socket_id = rte_eth_dev_socket_id(pt_id);
 590                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 591                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 592                                 rte_exit(EXIT_FAILURE,
 593                                          "Total sockets greater than %u\n",
 594                                          RTE_MAX_NUMA_NODES);
 595                         }
 596                         socket_ids[num_sockets++] = socket_id;
 597                 }
 598         }
 599
 600         nb_cfg_ports = nb_ports;
 601         nb_fwd_ports = nb_ports;
 602 }
 603
 604 void
 605 set_def_fwd_config(void)
 606 {
 607         set_default_fwd_lcores_config();
 608         set_def_peer_eth_addrs();
 609         set_default_fwd_ports_config();
 610 }
 611
 612 /* extremely pessimistic estimation of memory required to create a mempool */
 613 static int
 614 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 615 {
 616         unsigned int n_pages, mbuf_per_pg, leftover;
 617         uint64_t total_mem, mbuf_mem, obj_sz;
 618
 619         /* there is no good way to predict how much space the mempool will
 620          * occupy because it will allocate chunks on the fly, and some of those
 621          * will come from default DPDK memory while some will come from our
 622          * external memory, so just assume 128MB will be enough for everyone.
 623          */
 624         uint64_t hdr_mem = 128 << 20;
 625
 626         /* account for possible non-contiguousness */
 627         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 628         if (obj_sz > pgsz) {
 629                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 630                 return -1;
 631         }
 632
 633         mbuf_per_pg = pgsz / obj_sz;
 634         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 635         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 636
 637         mbuf_mem = n_pages * pgsz;
 638
 639         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 640
 641         if (total_mem > SIZE_MAX) {
 642                 TESTPMD_LOG(ERR, "Memory size too big\n");
 643                 return -1;
 644         }
 645         *out = (size_t)total_mem;
 646
 647         return 0;
 648 }
 649
 650 static int
 651 pagesz_flags(uint64_t page_sz)
 652 {
 653         /* as per mmap() manpage, all page sizes are log2 of page size
 654          * shifted by MAP_HUGE_SHIFT
 655          */
 656         int log2 = rte_log2_u64(page_sz);
 657
 658         return (log2 << HUGE_SHIFT);
 659 }
 660
 661 static void *
 662 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 663 {
 664         void *addr;
 665         int flags;
 666
 667         /* allocate anonymous hugepages */
 668         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 669         if (huge)
 670                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 671
 672         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 673         if (addr == MAP_FAILED)
 674                 return NULL;
 675
 676         return addr;
 677 }
 678
 679 struct extmem_param {
 680         void *addr;
 681         size_t len;
 682         size_t pgsz;
 683         rte_iova_t *iova_table;
 684         unsigned int iova_table_len;
 685 };
 686
 687 static int
 688 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 689                 bool huge)
 690 {
 691         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 692                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 693         unsigned int cur_page, n_pages, pgsz_idx;
 694         size_t mem_sz, cur_pgsz;
 695         rte_iova_t *iovas = NULL;
 696         void *addr;
 697         int ret;
 698
 699         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 700                 /* skip anything that is too big */
 701                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 702                         continue;
 703
 704                 cur_pgsz = pgsizes[pgsz_idx];
 705
 706                 /* if we were told not to allocate hugepages, override */
 707                 if (!huge)
 708                         cur_pgsz = sysconf(_SC_PAGESIZE);
 709
 710                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 711                 if (ret < 0) {
 712                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 713                         return -1;
 714                 }
 715
 716                 /* allocate our memory */
 717                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 718
 719                 /* if we couldn't allocate memory with a specified page size,
 720                  * that doesn't mean we can't do it with other page sizes, so
 721                  * try another one.
 722                  */
 723                 if (addr == NULL)
 724                         continue;
 725
 726                 /* store IOVA addresses for every page in this memory area */
 727                 n_pages = mem_sz / cur_pgsz;
 728
 729                 iovas = malloc(sizeof(*iovas) * n_pages);
 730
 731                 if (iovas == NULL) {
 732                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 733                         goto fail;
 734                 }
 735                 /* lock memory if it's not huge pages */
 736                 if (!huge)
 737                         mlock(addr, mem_sz);
 738
 739                 /* populate IOVA addresses */
 740                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 741                         rte_iova_t iova;
 742                         size_t offset;
 743                         void *cur;
 744
 745                         offset = cur_pgsz * cur_page;
 746                         cur = RTE_PTR_ADD(addr, offset);
 747
 748                         /* touch the page before getting its IOVA */
 749                         *(volatile char *)cur = 0;
 750
 751                         iova = rte_mem_virt2iova(cur);
 752
 753                         iovas[cur_page] = iova;
 754                 }
 755
 756                 break;
 757         }
 758         /* if we couldn't allocate anything */
 759         if (iovas == NULL)
 760                 return -1;
 761
 762         param->addr = addr;
 763         param->len = mem_sz;
 764         param->pgsz = cur_pgsz;
 765         param->iova_table = iovas;
 766         param->iova_table_len = n_pages;
 767
 768         return 0;
 769 fail:
 770         if (iovas)
 771                 free(iovas);
 772         if (addr)
 773                 munmap(addr, mem_sz);
 774
 775         return -1;
 776 }
 777
 778 static int
 779 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 780 {
 781         struct extmem_param param;
 782         int socket_id, ret;
 783
 784         memset(&param, 0, sizeof(param));
 785
 786         /* check if our heap exists */
 787         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 788         if (socket_id < 0) {
 789                 /* create our heap */
 790                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 791                 if (ret < 0) {
 792                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 793                         return -1;
 794                 }
 795         }
 796
 797         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 798         if (ret < 0) {
 799                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 800                 return -1;
 801         }
 802
 803         /* we now have a valid memory area, so add it to heap */
 804         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 805                         param.addr, param.len, param.iova_table,
 806                         param.iova_table_len, param.pgsz);
 807
 808         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 809
 810         /* not needed any more */
 811         free(param.iova_table);
 812
 813         if (ret < 0) {
 814                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 815                 munmap(param.addr, param.len);
 816                 return -1;
 817         }
 818
 819         /* success */
 820
 821         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 822                         param.len >> 20);
 823
 824         return 0;
 825 }
 826 static void
 827 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 828              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 829 {
 830         uint16_t pid = 0;
 831         int ret;
 832
 833         RTE_ETH_FOREACH_DEV(pid) {
 834                 struct rte_eth_dev *dev =
 835                         &rte_eth_devices[pid];
 836
 837                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 838                                         memhdr->len);
 839                 if (ret) {
 840                         TESTPMD_LOG(DEBUG,
 841                                     "unable to DMA unmap addr 0x%p "
 842                                     "for device %s\n",
 843                                     memhdr->addr, dev->data->name);
 844                 }
 845         }
 846         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 847         if (ret) {
 848                 TESTPMD_LOG(DEBUG,
 849                             "unable to un-register addr 0x%p\n", memhdr->addr);
 850         }
 851 }
 852
 853 static void
 854 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 855            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 856 {
 857         uint16_t pid = 0;
 858         size_t page_size = sysconf(_SC_PAGESIZE);
 859         int ret;
 860
 861         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 862                                   page_size);
 863         if (ret) {
 864                 TESTPMD_LOG(DEBUG,
 865                             "unable to register addr 0x%p\n", memhdr->addr);
 866                 return;
 867         }
 868         RTE_ETH_FOREACH_DEV(pid) {
 869                 struct rte_eth_dev *dev =
 870                         &rte_eth_devices[pid];
 871
 872                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 873                                       memhdr->len);
 874                 if (ret) {
 875                         TESTPMD_LOG(DEBUG,
 876                                     "unable to DMA map addr 0x%p "
 877                                     "for device %s\n",
 878                                     memhdr->addr, dev->data->name);
 879                 }
 880         }
 881 }
 882
 883 static unsigned int
 884 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 885             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 886 {
 887         struct rte_pktmbuf_extmem *xmem;
 888         unsigned int ext_num, zone_num, elt_num;
 889         uint16_t elt_size;
 890
 891         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 892         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 893         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 894
 895         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 896         if (xmem == NULL) {
 897                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 898                                  "external buffer descriptors\n");
 899                 *ext_mem = NULL;
 900                 return 0;
 901         }
 902         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 903                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 904                 const struct rte_memzone *mz;
 905                 char mz_name[RTE_MEMZONE_NAMESIZE];
 906                 int ret;
 907
 908                 ret = snprintf(mz_name, sizeof(mz_name),
 909                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 910                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 911                         errno = ENAMETOOLONG;
 912                         ext_num = 0;
 913                         break;
 914                 }
 915                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 916                                                  socket_id,
 917                                                  RTE_MEMZONE_IOVA_CONTIG |
 918                                                  RTE_MEMZONE_1GB |
 919                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 920                                                  EXTBUF_ZONE_SIZE);
 921                 if (mz == NULL) {
 922                         /*
 923                          * The caller exits on external buffer creation
 924                          * error, so there is no need to free memzones.
 925                          */
 926                         errno = ENOMEM;
 927                         ext_num = 0;
 928                         break;
 929                 }
 930                 xseg->buf_ptr = mz->addr;
 931                 xseg->buf_iova = mz->iova;
 932                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 933                 xseg->elt_size = elt_size;
 934         }
 935         if (ext_num == 0 && xmem != NULL) {
 936                 free(xmem);
 937                 xmem = NULL;
 938         }
 939         *ext_mem = xmem;
 940         return ext_num;
 941 }
 942
 943 /*
 944  * Configuration initialisation done once at init time.
 945  */
 946 static struct rte_mempool *
 947 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 948                  unsigned int socket_id)
 949 {
 950         char pool_name[RTE_MEMPOOL_NAMESIZE];
 951         struct rte_mempool *rte_mp = NULL;
 952         uint32_t mb_size;
 953
 954         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 955         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 956
 957         TESTPMD_LOG(INFO,
 958                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 959                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 960
 961         switch (mp_alloc_type) {
 962         case MP_ALLOC_NATIVE:
 963                 {
 964                         /* wrapper to rte_mempool_create() */
 965                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 966                                         rte_mbuf_best_mempool_ops());
 967                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 968                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 969                         break;
 970                 }
 971         case MP_ALLOC_ANON:
 972                 {
 973                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 974                                 mb_size, (unsigned int) mb_mempool_cache,
 975                                 sizeof(struct rte_pktmbuf_pool_private),
 976                                 socket_id, mempool_flags);
 977                         if (rte_mp == NULL)
 978                                 goto err;
 979
 980                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 981                                 rte_mempool_free(rte_mp);
 982                                 rte_mp = NULL;
 983                                 goto err;
 984                         }
 985                         rte_pktmbuf_pool_init(rte_mp, NULL);
 986                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 987                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 988                         break;
 989                 }
 990         case MP_ALLOC_XMEM:
 991         case MP_ALLOC_XMEM_HUGE:
 992                 {
 993                         int heap_socket;
 994                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 995
 996                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 997                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 998
 999                         heap_socket =
1000                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1001                         if (heap_socket < 0)
1002                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1003
1004                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1005                                         rte_mbuf_best_mempool_ops());
1006                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1007                                         mb_mempool_cache, 0, mbuf_seg_size,
1008                                         heap_socket);
1009                         break;
1010                 }
1011         case MP_ALLOC_XBUF:
1012                 {
1013                         struct rte_pktmbuf_extmem *ext_mem;
1014                         unsigned int ext_num;
1015
1016                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1017                                                socket_id, pool_name, &ext_mem);
1018                         if (ext_num == 0)
1019                                 rte_exit(EXIT_FAILURE,
1020                                          "Can't create pinned data buffers\n");
1021
1022                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1023                                         rte_mbuf_best_mempool_ops());
1024                         rte_mp = rte_pktmbuf_pool_create_extbuf
1025                                         (pool_name, nb_mbuf, mb_mempool_cache,
1026                                          0, mbuf_seg_size, socket_id,
1027                                          ext_mem, ext_num);
1028                         free(ext_mem);
1029                         break;
1030                 }
1031         default:
1032                 {
1033                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1034                 }
1035         }
1036
1037 err:
1038         if (rte_mp == NULL) {
1039                 rte_exit(EXIT_FAILURE,
1040                         "Creation of mbuf pool for socket %u failed: %s\n",
1041                         socket_id, rte_strerror(rte_errno));
1042         } else if (verbose_level > 0) {
1043                 rte_mempool_dump(stdout, rte_mp);
1044         }
1045         return rte_mp;
1046 }
1047
1048 /*
1049  * Check given socket id is valid or not with NUMA mode,
1050  * if valid, return 0, else return -1
1051  */
1052 static int
1053 check_socket_id(const unsigned int socket_id)
1054 {
1055         static int warning_once = 0;
1056
1057         if (new_socket_id(socket_id)) {
1058                 if (!warning_once && numa_support)
1059                         printf("Warning: NUMA should be configured manually by"
1060                                " using --port-numa-config and"
1061                                " --ring-numa-config parameters along with"
1062                                " --numa.\n");
1063                 warning_once = 1;
1064                 return -1;
1065         }
1066         return 0;
1067 }
1068
1069 /*
1070  * Get the allowed maximum number of RX queues.
1071  * *pid return the port id which has minimal value of
1072  * max_rx_queues in all ports.
1073  */
1074 queueid_t
1075 get_allowed_max_nb_rxq(portid_t *pid)
1076 {
1077         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1078         bool max_rxq_valid = false;
1079         portid_t pi;
1080         struct rte_eth_dev_info dev_info;
1081
1082         RTE_ETH_FOREACH_DEV(pi) {
1083                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1084                         continue;
1085
1086                 max_rxq_valid = true;
1087                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1088                         allowed_max_rxq = dev_info.max_rx_queues;
1089                         *pid = pi;
1090                 }
1091         }
1092         return max_rxq_valid ? allowed_max_rxq : 0;
1093 }
1094
1095 /*
1096  * Check input rxq is valid or not.
1097  * If input rxq is not greater than any of maximum number
1098  * of RX queues of all ports, it is valid.
1099  * if valid, return 0, else return -1
1100  */
1101 int
1102 check_nb_rxq(queueid_t rxq)
1103 {
1104         queueid_t allowed_max_rxq;
1105         portid_t pid = 0;
1106
1107         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1108         if (rxq > allowed_max_rxq) {
1109                 printf("Fail: input rxq (%u) can't be greater "
1110                        "than max_rx_queues (%u) of port %u\n",
1111                        rxq,
1112                        allowed_max_rxq,
1113                        pid);
1114                 return -1;
1115         }
1116         return 0;
1117 }
1118
1119 /*
1120  * Get the allowed maximum number of TX queues.
1121  * *pid return the port id which has minimal value of
1122  * max_tx_queues in all ports.
1123  */
1124 queueid_t
1125 get_allowed_max_nb_txq(portid_t *pid)
1126 {
1127         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1128         bool max_txq_valid = false;
1129         portid_t pi;
1130         struct rte_eth_dev_info dev_info;
1131
1132         RTE_ETH_FOREACH_DEV(pi) {
1133                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1134                         continue;
1135
1136                 max_txq_valid = true;
1137                 if (dev_info.max_tx_queues < allowed_max_txq) {
1138                         allowed_max_txq = dev_info.max_tx_queues;
1139                         *pid = pi;
1140                 }
1141         }
1142         return max_txq_valid ? allowed_max_txq : 0;
1143 }
1144
1145 /*
1146  * Check input txq is valid or not.
1147  * If input txq is not greater than any of maximum number
1148  * of TX queues of all ports, it is valid.
1149  * if valid, return 0, else return -1
1150  */
1151 int
1152 check_nb_txq(queueid_t txq)
1153 {
1154         queueid_t allowed_max_txq;
1155         portid_t pid = 0;
1156
1157         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1158         if (txq > allowed_max_txq) {
1159                 printf("Fail: input txq (%u) can't be greater "
1160                        "than max_tx_queues (%u) of port %u\n",
1161                        txq,
1162                        allowed_max_txq,
1163                        pid);
1164                 return -1;
1165         }
1166         return 0;
1167 }
1168
1169 /*
1170  * Get the allowed maximum number of RXDs of every rx queue.
1171  * *pid return the port id which has minimal value of
1172  * max_rxd in all queues of all ports.
1173  */
1174 static uint16_t
1175 get_allowed_max_nb_rxd(portid_t *pid)
1176 {
1177         uint16_t allowed_max_rxd = UINT16_MAX;
1178         portid_t pi;
1179         struct rte_eth_dev_info dev_info;
1180
1181         RTE_ETH_FOREACH_DEV(pi) {
1182                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1183                         continue;
1184
1185                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1186                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1187                         *pid = pi;
1188                 }
1189         }
1190         return allowed_max_rxd;
1191 }
1192
1193 /*
1194  * Get the allowed minimal number of RXDs of every rx queue.
1195  * *pid return the port id which has minimal value of
1196  * min_rxd in all queues of all ports.
1197  */
1198 static uint16_t
1199 get_allowed_min_nb_rxd(portid_t *pid)
1200 {
1201         uint16_t allowed_min_rxd = 0;
1202         portid_t pi;
1203         struct rte_eth_dev_info dev_info;
1204
1205         RTE_ETH_FOREACH_DEV(pi) {
1206                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1207                         continue;
1208
1209                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1210                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1211                         *pid = pi;
1212                 }
1213         }
1214
1215         return allowed_min_rxd;
1216 }
1217
1218 /*
1219  * Check input rxd is valid or not.
1220  * If input rxd is not greater than any of maximum number
1221  * of RXDs of every Rx queues and is not less than any of
1222  * minimal number of RXDs of every Rx queues, it is valid.
1223  * if valid, return 0, else return -1
1224  */
1225 int
1226 check_nb_rxd(queueid_t rxd)
1227 {
1228         uint16_t allowed_max_rxd;
1229         uint16_t allowed_min_rxd;
1230         portid_t pid = 0;
1231
1232         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1233         if (rxd > allowed_max_rxd) {
1234                 printf("Fail: input rxd (%u) can't be greater "
1235                        "than max_rxds (%u) of port %u\n",
1236                        rxd,
1237                        allowed_max_rxd,
1238                        pid);
1239                 return -1;
1240         }
1241
1242         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1243         if (rxd < allowed_min_rxd) {
1244                 printf("Fail: input rxd (%u) can't be less "
1245                        "than min_rxds (%u) of port %u\n",
1246                        rxd,
1247                        allowed_min_rxd,
1248                        pid);
1249                 return -1;
1250         }
1251
1252         return 0;
1253 }
1254
1255 /*
1256  * Get the allowed maximum number of TXDs of every rx queues.
1257  * *pid return the port id which has minimal value of
1258  * max_txd in every tx queue.
1259  */
1260 static uint16_t
1261 get_allowed_max_nb_txd(portid_t *pid)
1262 {
1263         uint16_t allowed_max_txd = UINT16_MAX;
1264         portid_t pi;
1265         struct rte_eth_dev_info dev_info;
1266
1267         RTE_ETH_FOREACH_DEV(pi) {
1268                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1269                         continue;
1270
1271                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1272                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1273                         *pid = pi;
1274                 }
1275         }
1276         return allowed_max_txd;
1277 }
1278
1279 /*
1280  * Get the allowed maximum number of TXDs of every tx queues.
1281  * *pid return the port id which has minimal value of
1282  * min_txd in every tx queue.
1283  */
1284 static uint16_t
1285 get_allowed_min_nb_txd(portid_t *pid)
1286 {
1287         uint16_t allowed_min_txd = 0;
1288         portid_t pi;
1289         struct rte_eth_dev_info dev_info;
1290
1291         RTE_ETH_FOREACH_DEV(pi) {
1292                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1293                         continue;
1294
1295                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1296                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1297                         *pid = pi;
1298                 }
1299         }
1300
1301         return allowed_min_txd;
1302 }
1303
1304 /*
1305  * Check input txd is valid or not.
1306  * If input txd is not greater than any of maximum number
1307  * of TXDs of every Rx queues, it is valid.
1308  * if valid, return 0, else return -1
1309  */
1310 int
1311 check_nb_txd(queueid_t txd)
1312 {
1313         uint16_t allowed_max_txd;
1314         uint16_t allowed_min_txd;
1315         portid_t pid = 0;
1316
1317         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1318         if (txd > allowed_max_txd) {
1319                 printf("Fail: input txd (%u) can't be greater "
1320                        "than max_txds (%u) of port %u\n",
1321                        txd,
1322                        allowed_max_txd,
1323                        pid);
1324                 return -1;
1325         }
1326
1327         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1328         if (txd < allowed_min_txd) {
1329                 printf("Fail: input txd (%u) can't be less "
1330                        "than min_txds (%u) of port %u\n",
1331                        txd,
1332                        allowed_min_txd,
1333                        pid);
1334                 return -1;
1335         }
1336         return 0;
1337 }
1338
1339
1340 /*
1341  * Get the allowed maximum number of hairpin queues.
1342  * *pid return the port id which has minimal value of
1343  * max_hairpin_queues in all ports.
1344  */
1345 queueid_t
1346 get_allowed_max_nb_hairpinq(portid_t *pid)
1347 {
1348         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1349         portid_t pi;
1350         struct rte_eth_hairpin_cap cap;
1351
1352         RTE_ETH_FOREACH_DEV(pi) {
1353                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1354                         *pid = pi;
1355                         return 0;
1356                 }
1357                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1358                         allowed_max_hairpinq = cap.max_nb_queues;
1359                         *pid = pi;
1360                 }
1361         }
1362         return allowed_max_hairpinq;
1363 }
1364
1365 /*
1366  * Check input hairpin is valid or not.
1367  * If input hairpin is not greater than any of maximum number
1368  * of hairpin queues of all ports, it is valid.
1369  * if valid, return 0, else return -1
1370  */
1371 int
1372 check_nb_hairpinq(queueid_t hairpinq)
1373 {
1374         queueid_t allowed_max_hairpinq;
1375         portid_t pid = 0;
1376
1377         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1378         if (hairpinq > allowed_max_hairpinq) {
1379                 printf("Fail: input hairpin (%u) can't be greater "
1380                        "than max_hairpin_queues (%u) of port %u\n",
1381                        hairpinq, allowed_max_hairpinq, pid);
1382                 return -1;
1383         }
1384         return 0;
1385 }
1386
1387 static void
1388 init_config(void)
1389 {
1390         portid_t pid;
1391         struct rte_port *port;
1392         struct rte_mempool *mbp;
1393         unsigned int nb_mbuf_per_pool;
1394         lcoreid_t  lc_id;
1395         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1396         struct rte_gro_param gro_param;
1397         uint32_t gso_types;
1398         uint16_t data_size;
1399         bool warning = 0;
1400         int k;
1401         int ret;
1402
1403         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1404
1405         /* Configuration of logical cores. */
1406         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1407                                 sizeof(struct fwd_lcore *) * nb_lcores,
1408                                 RTE_CACHE_LINE_SIZE);
1409         if (fwd_lcores == NULL) {
1410                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1411                                                         "failed\n", nb_lcores);
1412         }
1413         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1414                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1415                                                sizeof(struct fwd_lcore),
1416                                                RTE_CACHE_LINE_SIZE);
1417                 if (fwd_lcores[lc_id] == NULL) {
1418                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1419                                                                 "failed\n");
1420                 }
1421                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1422         }
1423
1424         RTE_ETH_FOREACH_DEV(pid) {
1425                 port = &ports[pid];
1426                 /* Apply default TxRx configuration for all ports */
1427                 port->dev_conf.txmode = tx_mode;
1428                 port->dev_conf.rxmode = rx_mode;
1429
1430                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1431                 if (ret != 0)
1432                         rte_exit(EXIT_FAILURE,
1433                                  "rte_eth_dev_info_get() failed\n");
1434
1435                 if (!(port->dev_info.tx_offload_capa &
1436                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1437                         port->dev_conf.txmode.offloads &=
1438                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1439                 if (numa_support) {
1440                         if (port_numa[pid] != NUMA_NO_CONFIG)
1441                                 port_per_socket[port_numa[pid]]++;
1442                         else {
1443                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1444
1445                                 /*
1446                                  * if socket_id is invalid,
1447                                  * set to the first available socket.
1448                                  */
1449                                 if (check_socket_id(socket_id) < 0)
1450                                         socket_id = socket_ids[0];
1451                                 port_per_socket[socket_id]++;
1452                         }
1453                 }
1454
1455                 /* Apply Rx offloads configuration */
1456                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1457                         port->rx_conf[k].offloads =
1458                                 port->dev_conf.rxmode.offloads;
1459                 /* Apply Tx offloads configuration */
1460                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1461                         port->tx_conf[k].offloads =
1462                                 port->dev_conf.txmode.offloads;
1463
1464                 /* set flag to initialize port/queue */
1465                 port->need_reconfig = 1;
1466                 port->need_reconfig_queues = 1;
1467                 port->tx_metadata = 0;
1468
1469                 /* Check for maximum number of segments per MTU. Accordingly
1470                  * update the mbuf data size.
1471                  */
1472                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1473                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1474                         data_size = rx_mode.max_rx_pkt_len /
1475                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1476
1477                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1478                                                         mbuf_data_size) {
1479                                 mbuf_data_size = data_size +
1480                                                  RTE_PKTMBUF_HEADROOM;
1481                                 warning = 1;
1482                         }
1483                 }
1484         }
1485
1486         if (warning)
1487                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1488                             mbuf_data_size);
1489
1490         /*
1491          * Create pools of mbuf.
1492          * If NUMA support is disabled, create a single pool of mbuf in
1493          * socket 0 memory by default.
1494          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1495          *
1496          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1497          * nb_txd can be configured at run time.
1498          */
1499         if (param_total_num_mbufs)
1500                 nb_mbuf_per_pool = param_total_num_mbufs;
1501         else {
1502                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1503                         (nb_lcores * mb_mempool_cache) +
1504                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1505                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1506         }
1507
1508         if (numa_support) {
1509                 uint8_t i;
1510
1511                 for (i = 0; i < num_sockets; i++)
1512                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1513                                                        nb_mbuf_per_pool,
1514                                                        socket_ids[i]);
1515         } else {
1516                 if (socket_num == UMA_NO_CONFIG)
1517                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1518                                                        nb_mbuf_per_pool, 0);
1519                 else
1520                         mempools[socket_num] = mbuf_pool_create
1521                                                         (mbuf_data_size,
1522                                                          nb_mbuf_per_pool,
1523                                                          socket_num);
1524         }
1525
1526         init_port_config();
1527
1528         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1529                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1530         /*
1531          * Records which Mbuf pool to use by each logical core, if needed.
1532          */
1533         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1534                 mbp = mbuf_pool_find(
1535                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1536
1537                 if (mbp == NULL)
1538                         mbp = mbuf_pool_find(0);
1539                 fwd_lcores[lc_id]->mbp = mbp;
1540                 /* initialize GSO context */
1541                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1542                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1543                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1544                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1545                         RTE_ETHER_CRC_LEN;
1546                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1547         }
1548
1549         /* Configuration of packet forwarding streams. */
1550         if (init_fwd_streams() < 0)
1551                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1552
1553         fwd_config_setup();
1554
1555         /* create a gro context for each lcore */
1556         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1557         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1558         gro_param.max_item_per_flow = MAX_PKT_BURST;
1559         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1560                 gro_param.socket_id = rte_lcore_to_socket_id(
1561                                 fwd_lcores_cpuids[lc_id]);
1562                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1563                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1564                         rte_exit(EXIT_FAILURE,
1565                                         "rte_gro_ctx_create() failed\n");
1566                 }
1567         }
1568 }
1569
1570
1571 void
1572 reconfig(portid_t new_port_id, unsigned socket_id)
1573 {
1574         struct rte_port *port;
1575         int ret;
1576
1577         /* Reconfiguration of Ethernet ports. */
1578         port = &ports[new_port_id];
1579
1580         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1581         if (ret != 0)
1582                 return;
1583
1584         /* set flag to initialize port/queue */
1585         port->need_reconfig = 1;
1586         port->need_reconfig_queues = 1;
1587         port->socket_id = socket_id;
1588
1589         init_port_config();
1590 }
1591
1592
1593 int
1594 init_fwd_streams(void)
1595 {
1596         portid_t pid;
1597         struct rte_port *port;
1598         streamid_t sm_id, nb_fwd_streams_new;
1599         queueid_t q;
1600
1601         /* set socket id according to numa or not */
1602         RTE_ETH_FOREACH_DEV(pid) {
1603                 port = &ports[pid];
1604                 if (nb_rxq > port->dev_info.max_rx_queues) {
1605                         printf("Fail: nb_rxq(%d) is greater than "
1606                                 "max_rx_queues(%d)\n", nb_rxq,
1607                                 port->dev_info.max_rx_queues);
1608                         return -1;
1609                 }
1610                 if (nb_txq > port->dev_info.max_tx_queues) {
1611                         printf("Fail: nb_txq(%d) is greater than "
1612                                 "max_tx_queues(%d)\n", nb_txq,
1613                                 port->dev_info.max_tx_queues);
1614                         return -1;
1615                 }
1616                 if (numa_support) {
1617                         if (port_numa[pid] != NUMA_NO_CONFIG)
1618                                 port->socket_id = port_numa[pid];
1619                         else {
1620                                 port->socket_id = rte_eth_dev_socket_id(pid);
1621
1622                                 /*
1623                                  * if socket_id is invalid,
1624                                  * set to the first available socket.
1625                                  */
1626                                 if (check_socket_id(port->socket_id) < 0)
1627                                         port->socket_id = socket_ids[0];
1628                         }
1629                 }
1630                 else {
1631                         if (socket_num == UMA_NO_CONFIG)
1632                                 port->socket_id = 0;
1633                         else
1634                                 port->socket_id = socket_num;
1635                 }
1636         }
1637
1638         q = RTE_MAX(nb_rxq, nb_txq);
1639         if (q == 0) {
1640                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1641                 return -1;
1642         }
1643         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1644         if (nb_fwd_streams_new == nb_fwd_streams)
1645                 return 0;
1646         /* clear the old */
1647         if (fwd_streams != NULL) {
1648                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1649                         if (fwd_streams[sm_id] == NULL)
1650                                 continue;
1651                         rte_free(fwd_streams[sm_id]);
1652                         fwd_streams[sm_id] = NULL;
1653                 }
1654                 rte_free(fwd_streams);
1655                 fwd_streams = NULL;
1656         }
1657
1658         /* init new */
1659         nb_fwd_streams = nb_fwd_streams_new;
1660         if (nb_fwd_streams) {
1661                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1662                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1663                         RTE_CACHE_LINE_SIZE);
1664                 if (fwd_streams == NULL)
1665                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1666                                  " (struct fwd_stream *)) failed\n",
1667                                  nb_fwd_streams);
1668
1669                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1670                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1671                                 " struct fwd_stream", sizeof(struct fwd_stream),
1672                                 RTE_CACHE_LINE_SIZE);
1673                         if (fwd_streams[sm_id] == NULL)
1674                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1675                                          "(struct fwd_stream) failed\n");
1676                 }
1677         }
1678
1679         return 0;
1680 }
1681
1682 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1683 static void
1684 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1685 {
1686         uint64_t total_burst, sburst;
1687         uint64_t nb_burst;
1688         uint64_t burst_stats[4];
1689         uint16_t pktnb_stats[4];
1690         uint16_t nb_pkt;
1691         int burst_percent[4], sburstp;
1692         int i;
1693
1694         /*
1695          * First compute the total number of packet bursts and the
1696          * two highest numbers of bursts of the same number of packets.
1697          */
1698         memset(&burst_stats, 0x0, sizeof(burst_stats));
1699         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1700
1701         /* Show stats for 0 burst size always */
1702         total_burst = pbs->pkt_burst_spread[0];
1703         burst_stats[0] = pbs->pkt_burst_spread[0];
1704         pktnb_stats[0] = 0;
1705
1706         /* Find the next 2 burst sizes with highest occurrences. */
1707         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1708                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1709
1710                 if (nb_burst == 0)
1711                         continue;
1712
1713                 total_burst += nb_burst;
1714
1715                 if (nb_burst > burst_stats[1]) {
1716                         burst_stats[2] = burst_stats[1];
1717                         pktnb_stats[2] = pktnb_stats[1];
1718                         burst_stats[1] = nb_burst;
1719                         pktnb_stats[1] = nb_pkt;
1720                 } else if (nb_burst > burst_stats[2]) {
1721                         burst_stats[2] = nb_burst;
1722                         pktnb_stats[2] = nb_pkt;
1723                 }
1724         }
1725         if (total_burst == 0)
1726                 return;
1727
1728         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1729         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1730                 if (i == 3) {
1731                         printf("%d%% of other]\n", 100 - sburstp);
1732                         return;
1733                 }
1734
1735                 sburst += burst_stats[i];
1736                 if (sburst == total_burst) {
1737                         printf("%d%% of %d pkts]\n",
1738                                 100 - sburstp, (int) pktnb_stats[i]);
1739                         return;
1740                 }
1741
1742                 burst_percent[i] =
1743                         (double)burst_stats[i] / total_burst * 100;
1744                 printf("%d%% of %d pkts + ",
1745                         burst_percent[i], (int) pktnb_stats[i]);
1746                 sburstp += burst_percent[i];
1747         }
1748 }
1749 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1750
1751 static void
1752 fwd_stream_stats_display(streamid_t stream_id)
1753 {
1754         struct fwd_stream *fs;
1755         static const char *fwd_top_stats_border = "-------";
1756
1757         fs = fwd_streams[stream_id];
1758         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1759             (fs->fwd_dropped == 0))
1760                 return;
1761         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1762                "TX Port=%2d/Queue=%2d %s\n",
1763                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1764                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1765         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1766                " TX-dropped: %-14"PRIu64,
1767                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1768
1769         /* if checksum mode */
1770         if (cur_fwd_eng == &csum_fwd_engine) {
1771                 printf("  RX- bad IP checksum: %-14"PRIu64
1772                        "  Rx- bad L4 checksum: %-14"PRIu64
1773                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1774                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1775                         fs->rx_bad_outer_l4_csum);
1776         } else {
1777                 printf("\n");
1778         }
1779
1780 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1781         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1782         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1783 #endif
1784 }
1785
1786 void
1787 fwd_stats_display(void)
1788 {
1789         static const char *fwd_stats_border = "----------------------";
1790         static const char *acc_stats_border = "+++++++++++++++";
1791         struct {
1792                 struct fwd_stream *rx_stream;
1793                 struct fwd_stream *tx_stream;
1794                 uint64_t tx_dropped;
1795                 uint64_t rx_bad_ip_csum;
1796                 uint64_t rx_bad_l4_csum;
1797                 uint64_t rx_bad_outer_l4_csum;
1798         } ports_stats[RTE_MAX_ETHPORTS];
1799         uint64_t total_rx_dropped = 0;
1800         uint64_t total_tx_dropped = 0;
1801         uint64_t total_rx_nombuf = 0;
1802         struct rte_eth_stats stats;
1803 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1804         uint64_t fwd_cycles = 0;
1805 #endif
1806         uint64_t total_recv = 0;
1807         uint64_t total_xmit = 0;
1808         struct rte_port *port;
1809         streamid_t sm_id;
1810         portid_t pt_id;
1811         int i;
1812
1813         memset(ports_stats, 0, sizeof(ports_stats));
1814
1815         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1816                 struct fwd_stream *fs = fwd_streams[sm_id];
1817
1818                 if (cur_fwd_config.nb_fwd_streams >
1819                     cur_fwd_config.nb_fwd_ports) {
1820                         fwd_stream_stats_display(sm_id);
1821                 } else {
1822                         ports_stats[fs->tx_port].tx_stream = fs;
1823                         ports_stats[fs->rx_port].rx_stream = fs;
1824                 }
1825
1826                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1827
1828                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1829                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1830                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1831                                 fs->rx_bad_outer_l4_csum;
1832
1833 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1834                 fwd_cycles += fs->core_cycles;
1835 #endif
1836         }
1837         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1838                 uint8_t j;
1839
1840                 pt_id = fwd_ports_ids[i];
1841                 port = &ports[pt_id];
1842
1843                 rte_eth_stats_get(pt_id, &stats);
1844                 stats.ipackets -= port->stats.ipackets;
1845                 stats.opackets -= port->stats.opackets;
1846                 stats.ibytes -= port->stats.ibytes;
1847                 stats.obytes -= port->stats.obytes;
1848                 stats.imissed -= port->stats.imissed;
1849                 stats.oerrors -= port->stats.oerrors;
1850                 stats.rx_nombuf -= port->stats.rx_nombuf;
1851
1852                 total_recv += stats.ipackets;
1853                 total_xmit += stats.opackets;
1854                 total_rx_dropped += stats.imissed;
1855                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1856                 total_tx_dropped += stats.oerrors;
1857                 total_rx_nombuf  += stats.rx_nombuf;
1858
1859                 printf("\n  %s Forward statistics for port %-2d %s\n",
1860                        fwd_stats_border, pt_id, fwd_stats_border);
1861
1862                 if (!port->rx_queue_stats_mapping_enabled &&
1863                     !port->tx_queue_stats_mapping_enabled) {
1864                         printf("  RX-packets: %-14"PRIu64
1865                                " RX-dropped: %-14"PRIu64
1866                                "RX-total: %-"PRIu64"\n",
1867                                stats.ipackets, stats.imissed,
1868                                stats.ipackets + stats.imissed);
1869
1870                         if (cur_fwd_eng == &csum_fwd_engine)
1871                                 printf("  Bad-ipcsum: %-14"PRIu64
1872                                        " Bad-l4csum: %-14"PRIu64
1873                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1874                                        ports_stats[pt_id].rx_bad_ip_csum,
1875                                        ports_stats[pt_id].rx_bad_l4_csum,
1876                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1877                         if (stats.ierrors + stats.rx_nombuf > 0) {
1878                                 printf("  RX-error: %-"PRIu64"\n",
1879                                        stats.ierrors);
1880                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1881                                        stats.rx_nombuf);
1882                         }
1883
1884                         printf("  TX-packets: %-14"PRIu64
1885                                " TX-dropped: %-14"PRIu64
1886                                "TX-total: %-"PRIu64"\n",
1887                                stats.opackets, ports_stats[pt_id].tx_dropped,
1888                                stats.opackets + ports_stats[pt_id].tx_dropped);
1889                 } else {
1890                         printf("  RX-packets:             %14"PRIu64
1891                                "    RX-dropped:%14"PRIu64
1892                                "    RX-total:%14"PRIu64"\n",
1893                                stats.ipackets, stats.imissed,
1894                                stats.ipackets + stats.imissed);
1895
1896                         if (cur_fwd_eng == &csum_fwd_engine)
1897                                 printf("  Bad-ipcsum:%14"PRIu64
1898                                        "    Bad-l4csum:%14"PRIu64
1899                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1900                                        ports_stats[pt_id].rx_bad_ip_csum,
1901                                        ports_stats[pt_id].rx_bad_l4_csum,
1902                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1903                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1904                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1905                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1906                                        stats.rx_nombuf);
1907                         }
1908
1909                         printf("  TX-packets:             %14"PRIu64
1910                                "    TX-dropped:%14"PRIu64
1911                                "    TX-total:%14"PRIu64"\n",
1912                                stats.opackets, ports_stats[pt_id].tx_dropped,
1913                                stats.opackets + ports_stats[pt_id].tx_dropped);
1914                 }
1915
1916 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1917                 if (ports_stats[pt_id].rx_stream)
1918                         pkt_burst_stats_display("RX",
1919                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1920                 if (ports_stats[pt_id].tx_stream)
1921                         pkt_burst_stats_display("TX",
1922                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1923 #endif
1924
1925                 if (port->rx_queue_stats_mapping_enabled) {
1926                         printf("\n");
1927                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1928                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1929                                        "     RX-errors:%14"PRIu64
1930                                        "    RX-bytes:%14"PRIu64"\n",
1931                                        j, stats.q_ipackets[j],
1932                                        stats.q_errors[j], stats.q_ibytes[j]);
1933                         }
1934                         printf("\n");
1935                 }
1936                 if (port->tx_queue_stats_mapping_enabled) {
1937                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1938                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1939                                        "                                 TX-bytes:%14"
1940                                        PRIu64"\n",
1941                                        j, stats.q_opackets[j],
1942                                        stats.q_obytes[j]);
1943                         }
1944                 }
1945
1946                 printf("  %s--------------------------------%s\n",
1947                        fwd_stats_border, fwd_stats_border);
1948         }
1949
1950         printf("\n  %s Accumulated forward statistics for all ports"
1951                "%s\n",
1952                acc_stats_border, acc_stats_border);
1953         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1954                "%-"PRIu64"\n"
1955                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1956                "%-"PRIu64"\n",
1957                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1958                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1959         if (total_rx_nombuf > 0)
1960                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1961         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1962                "%s\n",
1963                acc_stats_border, acc_stats_border);
1964 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1965 #define CYC_PER_MHZ 1E6
1966         if (total_recv > 0 || total_xmit > 0) {
1967                 uint64_t total_pkts = 0;
1968                 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1969                     strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1970                         total_pkts = total_xmit;
1971                 else
1972                         total_pkts = total_recv;
1973
1974                 printf("\n  CPU cycles/packet=%.2F (total cycles="
1975                        "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1976                        " MHz Clock\n",
1977                        (double) fwd_cycles / total_pkts,
1978                        fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1979                        (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1980         }
1981 #endif
1982 }
1983
1984 void
1985 fwd_stats_reset(void)
1986 {
1987         streamid_t sm_id;
1988         portid_t pt_id;
1989         int i;
1990
1991         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1992                 pt_id = fwd_ports_ids[i];
1993                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1994         }
1995         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1996                 struct fwd_stream *fs = fwd_streams[sm_id];
1997
1998                 fs->rx_packets = 0;
1999                 fs->tx_packets = 0;
2000                 fs->fwd_dropped = 0;
2001                 fs->rx_bad_ip_csum = 0;
2002                 fs->rx_bad_l4_csum = 0;
2003                 fs->rx_bad_outer_l4_csum = 0;
2004
2005 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
2006                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2007                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2008 #endif
2009 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
2010                 fs->core_cycles = 0;
2011 #endif
2012         }
2013 }
2014
2015 static void
2016 flush_fwd_rx_queues(void)
2017 {
2018         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2019         portid_t  rxp;
2020         portid_t port_id;
2021         queueid_t rxq;
2022         uint16_t  nb_rx;
2023         uint16_t  i;
2024         uint8_t   j;
2025         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2026         uint64_t timer_period;
2027
2028         /* convert to number of cycles */
2029         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2030
2031         for (j = 0; j < 2; j++) {
2032                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2033                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2034                                 port_id = fwd_ports_ids[rxp];
2035                                 /**
2036                                 * testpmd can stuck in the below do while loop
2037                                 * if rte_eth_rx_burst() always returns nonzero
2038                                 * packets. So timer is added to exit this loop
2039                                 * after 1sec timer expiry.
2040                                 */
2041                                 prev_tsc = rte_rdtsc();
2042                                 do {
2043                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2044                                                 pkts_burst, MAX_PKT_BURST);
2045                                         for (i = 0; i < nb_rx; i++)
2046                                                 rte_pktmbuf_free(pkts_burst[i]);
2047
2048                                         cur_tsc = rte_rdtsc();
2049                                         diff_tsc = cur_tsc - prev_tsc;
2050                                         timer_tsc += diff_tsc;
2051                                 } while ((nb_rx > 0) &&
2052                                         (timer_tsc < timer_period));
2053                                 timer_tsc = 0;
2054                         }
2055                 }
2056                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2057         }
2058 }
2059
2060 static void
2061 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2062 {
2063         struct fwd_stream **fsm;
2064         streamid_t nb_fs;
2065         streamid_t sm_id;
2066 #ifdef RTE_LIBRTE_BITRATE
2067         uint64_t tics_per_1sec;
2068         uint64_t tics_datum;
2069         uint64_t tics_current;
2070         uint16_t i, cnt_ports;
2071
2072         cnt_ports = nb_ports;
2073         tics_datum = rte_rdtsc();
2074         tics_per_1sec = rte_get_timer_hz();
2075 #endif
2076         fsm = &fwd_streams[fc->stream_idx];
2077         nb_fs = fc->stream_nb;
2078         do {
2079                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2080                         (*pkt_fwd)(fsm[sm_id]);
2081 #ifdef RTE_LIBRTE_BITRATE
2082                 if (bitrate_enabled != 0 &&
2083                                 bitrate_lcore_id == rte_lcore_id()) {
2084                         tics_current = rte_rdtsc();
2085                         if (tics_current - tics_datum >= tics_per_1sec) {
2086                                 /* Periodic bitrate calculation */
2087                                 for (i = 0; i < cnt_ports; i++)
2088                                         rte_stats_bitrate_calc(bitrate_data,
2089                                                 ports_ids[i]);
2090                                 tics_datum = tics_current;
2091                         }
2092                 }
2093 #endif
2094 #ifdef RTE_LIBRTE_LATENCY_STATS
2095                 if (latencystats_enabled != 0 &&
2096                                 latencystats_lcore_id == rte_lcore_id())
2097                         rte_latencystats_update();
2098 #endif
2099
2100         } while (! fc->stopped);
2101 }
2102
2103 static int
2104 start_pkt_forward_on_core(void *fwd_arg)
2105 {
2106         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2107                              cur_fwd_config.fwd_eng->packet_fwd);
2108         return 0;
2109 }
2110
2111 /*
2112  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2113  * Used to start communication flows in network loopback test configurations.
2114  */
2115 static int
2116 run_one_txonly_burst_on_core(void *fwd_arg)
2117 {
2118         struct fwd_lcore *fwd_lc;
2119         struct fwd_lcore tmp_lcore;
2120
2121         fwd_lc = (struct fwd_lcore *) fwd_arg;
2122         tmp_lcore = *fwd_lc;
2123         tmp_lcore.stopped = 1;
2124         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2125         return 0;
2126 }
2127
2128 /*
2129  * Launch packet forwarding:
2130  *     - Setup per-port forwarding context.
2131  *     - launch logical cores with their forwarding configuration.
2132  */
2133 static void
2134 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2135 {
2136         port_fwd_begin_t port_fwd_begin;
2137         unsigned int i;
2138         unsigned int lc_id;
2139         int diag;
2140
2141         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2142         if (port_fwd_begin != NULL) {
2143                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2144                         (*port_fwd_begin)(fwd_ports_ids[i]);
2145         }
2146         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2147                 lc_id = fwd_lcores_cpuids[i];
2148                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2149                         fwd_lcores[i]->stopped = 0;
2150                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2151                                                      fwd_lcores[i], lc_id);
2152                         if (diag != 0)
2153                                 printf("launch lcore %u failed - diag=%d\n",
2154                                        lc_id, diag);
2155                 }
2156         }
2157 }
2158
2159 /*
2160  * Launch packet forwarding configuration.
2161  */
2162 void
2163 start_packet_forwarding(int with_tx_first)
2164 {
2165         port_fwd_begin_t port_fwd_begin;
2166         port_fwd_end_t  port_fwd_end;
2167         struct rte_port *port;
2168         unsigned int i;
2169         portid_t   pt_id;
2170
2171         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2172                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2173
2174         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2175                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2176
2177         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2178                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2179                 (!nb_rxq || !nb_txq))
2180                 rte_exit(EXIT_FAILURE,
2181                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2182                         cur_fwd_eng->fwd_mode_name);
2183
2184         if (all_ports_started() == 0) {
2185                 printf("Not all ports were started\n");
2186                 return;
2187         }
2188         if (test_done == 0) {
2189                 printf("Packet forwarding already started\n");
2190                 return;
2191         }
2192
2193
2194         if(dcb_test) {
2195                 for (i = 0; i < nb_fwd_ports; i++) {
2196                         pt_id = fwd_ports_ids[i];
2197                         port = &ports[pt_id];
2198                         if (!port->dcb_flag) {
2199                                 printf("In DCB mode, all forwarding ports must "
2200                                        "be configured in this mode.\n");
2201                                 return;
2202                         }
2203                 }
2204                 if (nb_fwd_lcores == 1) {
2205                         printf("In DCB mode,the nb forwarding cores "
2206                                "should be larger than 1.\n");
2207                         return;
2208                 }
2209         }
2210         test_done = 0;
2211
2212         fwd_config_setup();
2213
2214         if(!no_flush_rx)
2215                 flush_fwd_rx_queues();
2216
2217         pkt_fwd_config_display(&cur_fwd_config);
2218         rxtx_config_display();
2219
2220         fwd_stats_reset();
2221         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2222                 pt_id = fwd_ports_ids[i];
2223                 port = &ports[pt_id];
2224                 map_port_queue_stats_mapping_registers(pt_id, port);
2225         }
2226         if (with_tx_first) {
2227                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2228                 if (port_fwd_begin != NULL) {
2229                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2230                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2231                 }
2232                 while (with_tx_first--) {
2233                         launch_packet_forwarding(
2234                                         run_one_txonly_burst_on_core);
2235                         rte_eal_mp_wait_lcore();
2236                 }
2237                 port_fwd_end = tx_only_engine.port_fwd_end;
2238                 if (port_fwd_end != NULL) {
2239                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2240                                 (*port_fwd_end)(fwd_ports_ids[i]);
2241                 }
2242         }
2243         launch_packet_forwarding(start_pkt_forward_on_core);
2244 }
2245
2246 void
2247 stop_packet_forwarding(void)
2248 {
2249         port_fwd_end_t port_fwd_end;
2250         lcoreid_t lc_id;
2251         portid_t pt_id;
2252         int i;
2253
2254         if (test_done) {
2255                 printf("Packet forwarding not started\n");
2256                 return;
2257         }
2258         printf("Telling cores to stop...");
2259         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2260                 fwd_lcores[lc_id]->stopped = 1;
2261         printf("\nWaiting for lcores to finish...\n");
2262         rte_eal_mp_wait_lcore();
2263         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2264         if (port_fwd_end != NULL) {
2265                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2266                         pt_id = fwd_ports_ids[i];
2267                         (*port_fwd_end)(pt_id);
2268                 }
2269         }
2270
2271         fwd_stats_display();
2272
2273         printf("\nDone.\n");
2274         test_done = 1;
2275 }
2276
2277 void
2278 dev_set_link_up(portid_t pid)
2279 {
2280         if (rte_eth_dev_set_link_up(pid) < 0)
2281                 printf("\nSet link up fail.\n");
2282 }
2283
2284 void
2285 dev_set_link_down(portid_t pid)
2286 {
2287         if (rte_eth_dev_set_link_down(pid) < 0)
2288                 printf("\nSet link down fail.\n");
2289 }
2290
2291 static int
2292 all_ports_started(void)
2293 {
2294         portid_t pi;
2295         struct rte_port *port;
2296
2297         RTE_ETH_FOREACH_DEV(pi) {
2298                 port = &ports[pi];
2299                 /* Check if there is a port which is not started */
2300                 if ((port->port_status != RTE_PORT_STARTED) &&
2301                         (port->slave_flag == 0))
2302                         return 0;
2303         }
2304
2305         /* No port is not started */
2306         return 1;
2307 }
2308
2309 int
2310 port_is_stopped(portid_t port_id)
2311 {
2312         struct rte_port *port = &ports[port_id];
2313
2314         if ((port->port_status != RTE_PORT_STOPPED) &&
2315             (port->slave_flag == 0))
2316                 return 0;
2317         return 1;
2318 }
2319
2320 int
2321 all_ports_stopped(void)
2322 {
2323         portid_t pi;
2324
2325         RTE_ETH_FOREACH_DEV(pi) {
2326                 if (!port_is_stopped(pi))
2327                         return 0;
2328         }
2329
2330         return 1;
2331 }
2332
2333 int
2334 port_is_started(portid_t port_id)
2335 {
2336         if (port_id_is_invalid(port_id, ENABLED_WARN))
2337                 return 0;
2338
2339         if (ports[port_id].port_status != RTE_PORT_STARTED)
2340                 return 0;
2341
2342         return 1;
2343 }
2344
2345 /* Configure the Rx and Tx hairpin queues for the selected port. */
2346 static int
2347 setup_hairpin_queues(portid_t pi)
2348 {
2349         queueid_t qi;
2350         struct rte_eth_hairpin_conf hairpin_conf = {
2351                 .peer_count = 1,
2352         };
2353         int i;
2354         int diag;
2355         struct rte_port *port = &ports[pi];
2356
2357         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2358                 hairpin_conf.peers[0].port = pi;
2359                 hairpin_conf.peers[0].queue = i + nb_rxq;
2360                 diag = rte_eth_tx_hairpin_queue_setup
2361                         (pi, qi, nb_txd, &hairpin_conf);
2362                 i++;
2363                 if (diag == 0)
2364                         continue;
2365
2366                 /* Fail to setup rx queue, return */
2367                 if (rte_atomic16_cmpset(&(port->port_status),
2368                                         RTE_PORT_HANDLING,
2369                                         RTE_PORT_STOPPED) == 0)
2370                         printf("Port %d can not be set back "
2371                                         "to stopped\n", pi);
2372                 printf("Fail to configure port %d hairpin "
2373                                 "queues\n", pi);
2374                 /* try to reconfigure queues next time */
2375                 port->need_reconfig_queues = 1;
2376                 return -1;
2377         }
2378         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2379                 hairpin_conf.peers[0].port = pi;
2380                 hairpin_conf.peers[0].queue = i + nb_txq;
2381                 diag = rte_eth_rx_hairpin_queue_setup
2382                         (pi, qi, nb_rxd, &hairpin_conf);
2383                 i++;
2384                 if (diag == 0)
2385                         continue;
2386
2387                 /* Fail to setup rx queue, return */
2388                 if (rte_atomic16_cmpset(&(port->port_status),
2389                                         RTE_PORT_HANDLING,
2390                                         RTE_PORT_STOPPED) == 0)
2391                         printf("Port %d can not be set back "
2392                                         "to stopped\n", pi);
2393                 printf("Fail to configure port %d hairpin "
2394                                 "queues\n", pi);
2395                 /* try to reconfigure queues next time */
2396                 port->need_reconfig_queues = 1;
2397                 return -1;
2398         }
2399         return 0;
2400 }
2401
2402 int
2403 start_port(portid_t pid)
2404 {
2405         int diag, need_check_link_status = -1;
2406         portid_t pi;
2407         queueid_t qi;
2408         struct rte_port *port;
2409         struct rte_ether_addr mac_addr;
2410         struct rte_eth_hairpin_cap cap;
2411
2412         if (port_id_is_invalid(pid, ENABLED_WARN))
2413                 return 0;
2414
2415         if(dcb_config)
2416                 dcb_test = 1;
2417         RTE_ETH_FOREACH_DEV(pi) {
2418                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2419                         continue;
2420
2421                 need_check_link_status = 0;
2422                 port = &ports[pi];
2423                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2424                                                  RTE_PORT_HANDLING) == 0) {
2425                         printf("Port %d is now not stopped\n", pi);
2426                         continue;
2427                 }
2428
2429                 if (port->need_reconfig > 0) {
2430                         port->need_reconfig = 0;
2431
2432                         if (flow_isolate_all) {
2433                                 int ret = port_flow_isolate(pi, 1);
2434                                 if (ret) {
2435                                         printf("Failed to apply isolated"
2436                                                " mode on port %d\n", pi);
2437                                         return -1;
2438                                 }
2439                         }
2440                         configure_rxtx_dump_callbacks(0);
2441                         printf("Configuring Port %d (socket %u)\n", pi,
2442                                         port->socket_id);
2443                         if (nb_hairpinq > 0 &&
2444                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2445                                 printf("Port %d doesn't support hairpin "
2446                                        "queues\n", pi);
2447                                 return -1;
2448                         }
2449                         /* configure port */
2450                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2451                                                      nb_txq + nb_hairpinq,
2452                                                      &(port->dev_conf));
2453                         if (diag != 0) {
2454                                 if (rte_atomic16_cmpset(&(port->port_status),
2455                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2456                                         printf("Port %d can not be set back "
2457                                                         "to stopped\n", pi);
2458                                 printf("Fail to configure port %d\n", pi);
2459                                 /* try to reconfigure port next time */
2460                                 port->need_reconfig = 1;
2461                                 return -1;
2462                         }
2463                 }
2464                 if (port->need_reconfig_queues > 0) {
2465                         port->need_reconfig_queues = 0;
2466                         /* setup tx queues */
2467                         for (qi = 0; qi < nb_txq; qi++) {
2468                                 if ((numa_support) &&
2469                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2470                                         diag = rte_eth_tx_queue_setup(pi, qi,
2471                                                 port->nb_tx_desc[qi],
2472                                                 txring_numa[pi],
2473                                                 &(port->tx_conf[qi]));
2474                                 else
2475                                         diag = rte_eth_tx_queue_setup(pi, qi,
2476                                                 port->nb_tx_desc[qi],
2477                                                 port->socket_id,
2478                                                 &(port->tx_conf[qi]));
2479
2480                                 if (diag == 0)
2481                                         continue;
2482
2483                                 /* Fail to setup tx queue, return */
2484                                 if (rte_atomic16_cmpset(&(port->port_status),
2485                                                         RTE_PORT_HANDLING,
2486                                                         RTE_PORT_STOPPED) == 0)
2487                                         printf("Port %d can not be set back "
2488                                                         "to stopped\n", pi);
2489                                 printf("Fail to configure port %d tx queues\n",
2490                                        pi);
2491                                 /* try to reconfigure queues next time */
2492                                 port->need_reconfig_queues = 1;
2493                                 return -1;
2494                         }
2495                         for (qi = 0; qi < nb_rxq; qi++) {
2496                                 /* setup rx queues */
2497                                 if ((numa_support) &&
2498                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2499                                         struct rte_mempool * mp =
2500                                                 mbuf_pool_find(rxring_numa[pi]);
2501                                         if (mp == NULL) {
2502                                                 printf("Failed to setup RX queue:"
2503                                                         "No mempool allocation"
2504                                                         " on the socket %d\n",
2505                                                         rxring_numa[pi]);
2506                                                 return -1;
2507                                         }
2508
2509                                         diag = rte_eth_rx_queue_setup(pi, qi,
2510                                              port->nb_rx_desc[qi],
2511                                              rxring_numa[pi],
2512                                              &(port->rx_conf[qi]),
2513                                              mp);
2514                                 } else {
2515                                         struct rte_mempool *mp =
2516                                                 mbuf_pool_find(port->socket_id);
2517                                         if (mp == NULL) {
2518                                                 printf("Failed to setup RX queue:"
2519                                                         "No mempool allocation"
2520                                                         " on the socket %d\n",
2521                                                         port->socket_id);
2522                                                 return -1;
2523                                         }
2524                                         diag = rte_eth_rx_queue_setup(pi, qi,
2525                                              port->nb_rx_desc[qi],
2526                                              port->socket_id,
2527                                              &(port->rx_conf[qi]),
2528                                              mp);
2529                                 }
2530                                 if (diag == 0)
2531                                         continue;
2532
2533                                 /* Fail to setup rx queue, return */
2534                                 if (rte_atomic16_cmpset(&(port->port_status),
2535                                                         RTE_PORT_HANDLING,
2536                                                         RTE_PORT_STOPPED) == 0)
2537                                         printf("Port %d can not be set back "
2538                                                         "to stopped\n", pi);
2539                                 printf("Fail to configure port %d rx queues\n",
2540                                        pi);
2541                                 /* try to reconfigure queues next time */
2542                                 port->need_reconfig_queues = 1;
2543                                 return -1;
2544                         }
2545                         /* setup hairpin queues */
2546                         if (setup_hairpin_queues(pi) != 0)
2547                                 return -1;
2548                 }
2549                 configure_rxtx_dump_callbacks(verbose_level);
2550                 if (clear_ptypes) {
2551                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2552                                         NULL, 0);
2553                         if (diag < 0)
2554                                 printf(
2555                                 "Port %d: Failed to disable Ptype parsing\n",
2556                                 pi);
2557                 }
2558
2559                 /* start port */
2560                 if (rte_eth_dev_start(pi) < 0) {
2561                         printf("Fail to start port %d\n", pi);
2562
2563                         /* Fail to setup rx queue, return */
2564                         if (rte_atomic16_cmpset(&(port->port_status),
2565                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2566                                 printf("Port %d can not be set back to "
2567                                                         "stopped\n", pi);
2568                         continue;
2569                 }
2570
2571                 if (rte_atomic16_cmpset(&(port->port_status),
2572                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2573                         printf("Port %d can not be set into started\n", pi);
2574
2575                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2576                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2577                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2578                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2579                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2580
2581                 /* at least one port started, need checking link status */
2582                 need_check_link_status = 1;
2583         }
2584
2585         if (need_check_link_status == 1 && !no_link_check)
2586                 check_all_ports_link_status(RTE_PORT_ALL);
2587         else if (need_check_link_status == 0)
2588                 printf("Please stop the ports first\n");
2589
2590         printf("Done\n");
2591         return 0;
2592 }
2593
2594 void
2595 stop_port(portid_t pid)
2596 {
2597         portid_t pi;
2598         struct rte_port *port;
2599         int need_check_link_status = 0;
2600
2601         if (dcb_test) {
2602                 dcb_test = 0;
2603                 dcb_config = 0;
2604         }
2605
2606         if (port_id_is_invalid(pid, ENABLED_WARN))
2607                 return;
2608
2609         printf("Stopping ports...\n");
2610
2611         RTE_ETH_FOREACH_DEV(pi) {
2612                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2613                         continue;
2614
2615                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2616                         printf("Please remove port %d from forwarding configuration.\n", pi);
2617                         continue;
2618                 }
2619
2620                 if (port_is_bonding_slave(pi)) {
2621                         printf("Please remove port %d from bonded device.\n", pi);
2622                         continue;
2623                 }
2624
2625                 port = &ports[pi];
2626                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2627                                                 RTE_PORT_HANDLING) == 0)
2628                         continue;
2629
2630                 rte_eth_dev_stop(pi);
2631
2632                 if (rte_atomic16_cmpset(&(port->port_status),
2633                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2634                         printf("Port %d can not be set into stopped\n", pi);
2635                 need_check_link_status = 1;
2636         }
2637         if (need_check_link_status && !no_link_check)
2638                 check_all_ports_link_status(RTE_PORT_ALL);
2639
2640         printf("Done\n");
2641 }
2642
2643 static void
2644 remove_invalid_ports_in(portid_t *array, portid_t *total)
2645 {
2646         portid_t i;
2647         portid_t new_total = 0;
2648
2649         for (i = 0; i < *total; i++)
2650                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2651                         array[new_total] = array[i];
2652                         new_total++;
2653                 }
2654         *total = new_total;
2655 }
2656
2657 static void
2658 remove_invalid_ports(void)
2659 {
2660         remove_invalid_ports_in(ports_ids, &nb_ports);
2661         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2662         nb_cfg_ports = nb_fwd_ports;
2663 }
2664
2665 void
2666 close_port(portid_t pid)
2667 {
2668         portid_t pi;
2669         struct rte_port *port;
2670
2671         if (port_id_is_invalid(pid, ENABLED_WARN))
2672                 return;
2673
2674         printf("Closing ports...\n");
2675
2676         RTE_ETH_FOREACH_DEV(pi) {
2677                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2678                         continue;
2679
2680                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2681                         printf("Please remove port %d from forwarding configuration.\n", pi);
2682                         continue;
2683                 }
2684
2685                 if (port_is_bonding_slave(pi)) {
2686                         printf("Please remove port %d from bonded device.\n", pi);
2687                         continue;
2688                 }
2689
2690                 port = &ports[pi];
2691                 if (rte_atomic16_cmpset(&(port->port_status),
2692                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2693                         printf("Port %d is already closed\n", pi);
2694                         continue;
2695                 }
2696
2697                 if (rte_atomic16_cmpset(&(port->port_status),
2698                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2699                         printf("Port %d is now not stopped\n", pi);
2700                         continue;
2701                 }
2702
2703                 if (port->flow_list)
2704                         port_flow_flush(pi);
2705                 rte_eth_dev_close(pi);
2706
2707                 remove_invalid_ports();
2708
2709                 if (rte_atomic16_cmpset(&(port->port_status),
2710                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2711                         printf("Port %d cannot be set to closed\n", pi);
2712         }
2713
2714         printf("Done\n");
2715 }
2716
2717 void
2718 reset_port(portid_t pid)
2719 {
2720         int diag;
2721         portid_t pi;
2722         struct rte_port *port;
2723
2724         if (port_id_is_invalid(pid, ENABLED_WARN))
2725                 return;
2726
2727         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2728                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2729                 printf("Can not reset port(s), please stop port(s) first.\n");
2730                 return;
2731         }
2732
2733         printf("Resetting ports...\n");
2734
2735         RTE_ETH_FOREACH_DEV(pi) {
2736                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2737                         continue;
2738
2739                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2740                         printf("Please remove port %d from forwarding "
2741                                "configuration.\n", pi);
2742                         continue;
2743                 }
2744
2745                 if (port_is_bonding_slave(pi)) {
2746                         printf("Please remove port %d from bonded device.\n",
2747                                pi);
2748                         continue;
2749                 }
2750
2751                 diag = rte_eth_dev_reset(pi);
2752                 if (diag == 0) {
2753                         port = &ports[pi];
2754                         port->need_reconfig = 1;
2755                         port->need_reconfig_queues = 1;
2756                 } else {
2757                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2758                 }
2759         }
2760
2761         printf("Done\n");
2762 }
2763
2764 void
2765 attach_port(char *identifier)
2766 {
2767         portid_t pi;
2768         struct rte_dev_iterator iterator;
2769
2770         printf("Attaching a new port...\n");
2771
2772         if (identifier == NULL) {
2773                 printf("Invalid parameters are specified\n");
2774                 return;
2775         }
2776
2777         if (rte_dev_probe(identifier) < 0) {
2778                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2779                 return;
2780         }
2781
2782         /* first attach mode: event */
2783         if (setup_on_probe_event) {
2784                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2785                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2786                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2787                                         ports[pi].need_setup != 0)
2788                                 setup_attached_port(pi);
2789                 return;
2790         }
2791
2792         /* second attach mode: iterator */
2793         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2794                 /* setup ports matching the devargs used for probing */
2795                 if (port_is_forwarding(pi))
2796                         continue; /* port was already attached before */
2797                 setup_attached_port(pi);
2798         }
2799 }
2800
2801 static void
2802 setup_attached_port(portid_t pi)
2803 {
2804         unsigned int socket_id;
2805         int ret;
2806
2807         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2808         /* if socket_id is invalid, set to the first available socket. */
2809         if (check_socket_id(socket_id) < 0)
2810                 socket_id = socket_ids[0];
2811         reconfig(pi, socket_id);
2812         ret = rte_eth_promiscuous_enable(pi);
2813         if (ret != 0)
2814                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2815                         pi, rte_strerror(-ret));
2816
2817         ports_ids[nb_ports++] = pi;
2818         fwd_ports_ids[nb_fwd_ports++] = pi;
2819         nb_cfg_ports = nb_fwd_ports;
2820         ports[pi].need_setup = 0;
2821         ports[pi].port_status = RTE_PORT_STOPPED;
2822
2823         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2824         printf("Done\n");
2825 }
2826
2827 static void
2828 detach_device(struct rte_device *dev)
2829 {
2830         portid_t sibling;
2831
2832         if (dev == NULL) {
2833                 printf("Device already removed\n");
2834                 return;
2835         }
2836
2837         printf("Removing a device...\n");
2838
2839         if (rte_dev_remove(dev) < 0) {
2840                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2841                 return;
2842         }
2843         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2844                 /* reset mapping between old ports and removed device */
2845                 rte_eth_devices[sibling].device = NULL;
2846                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2847                         /* sibling ports are forced to be closed */
2848                         ports[sibling].port_status = RTE_PORT_CLOSED;
2849                         printf("Port %u is closed\n", sibling);
2850                 }
2851         }
2852
2853         remove_invalid_ports();
2854
2855         printf("Device is detached\n");
2856         printf("Now total ports is %d\n", nb_ports);
2857         printf("Done\n");
2858         return;
2859 }
2860
2861 void
2862 detach_port_device(portid_t port_id)
2863 {
2864         if (port_id_is_invalid(port_id, ENABLED_WARN))
2865                 return;
2866
2867         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2868                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2869                         printf("Port not stopped\n");
2870                         return;
2871                 }
2872                 printf("Port was not closed\n");
2873                 if (ports[port_id].flow_list)
2874                         port_flow_flush(port_id);
2875         }
2876
2877         detach_device(rte_eth_devices[port_id].device);
2878 }
2879
2880 void
2881 detach_devargs(char *identifier)
2882 {
2883         struct rte_dev_iterator iterator;
2884         struct rte_devargs da;
2885         portid_t port_id;
2886
2887         printf("Removing a device...\n");
2888
2889         memset(&da, 0, sizeof(da));
2890         if (rte_devargs_parsef(&da, "%s", identifier)) {
2891                 printf("cannot parse identifier\n");
2892                 if (da.args)
2893                         free(da.args);
2894                 return;
2895         }
2896
2897         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2898                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2899                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2900                                 printf("Port %u not stopped\n", port_id);
2901                                 rte_eth_iterator_cleanup(&iterator);
2902                                 return;
2903                         }
2904
2905                         /* sibling ports are forced to be closed */
2906                         if (ports[port_id].flow_list)
2907                                 port_flow_flush(port_id);
2908                         ports[port_id].port_status = RTE_PORT_CLOSED;
2909                         printf("Port %u is now closed\n", port_id);
2910                 }
2911         }
2912
2913         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2914                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2915                             da.name, da.bus->name);
2916                 return;
2917         }
2918
2919         remove_invalid_ports();
2920
2921         printf("Device %s is detached\n", identifier);
2922         printf("Now total ports is %d\n", nb_ports);
2923         printf("Done\n");
2924 }
2925
2926 void
2927 pmd_test_exit(void)
2928 {
2929         portid_t pt_id;
2930         int ret;
2931         int i;
2932
2933         if (test_done == 0)
2934                 stop_packet_forwarding();
2935
2936         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2937                 if (mempools[i]) {
2938                         if (mp_alloc_type == MP_ALLOC_ANON)
2939                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2940                                                      NULL);
2941                 }
2942         }
2943         if (ports != NULL) {
2944                 no_link_check = 1;
2945                 RTE_ETH_FOREACH_DEV(pt_id) {
2946                         printf("\nStopping port %d...\n", pt_id);
2947                         fflush(stdout);
2948                         stop_port(pt_id);
2949                 }
2950                 RTE_ETH_FOREACH_DEV(pt_id) {
2951                         printf("\nShutting down port %d...\n", pt_id);
2952                         fflush(stdout);
2953                         close_port(pt_id);
2954                 }
2955         }
2956
2957         if (hot_plug) {
2958                 ret = rte_dev_event_monitor_stop();
2959                 if (ret) {
2960                         RTE_LOG(ERR, EAL,
2961                                 "fail to stop device event monitor.");
2962                         return;
2963                 }
2964
2965                 ret = rte_dev_event_callback_unregister(NULL,
2966                         dev_event_callback, NULL);
2967                 if (ret < 0) {
2968                         RTE_LOG(ERR, EAL,
2969                                 "fail to unregister device event callback.\n");
2970                         return;
2971                 }
2972
2973                 ret = rte_dev_hotplug_handle_disable();
2974                 if (ret) {
2975                         RTE_LOG(ERR, EAL,
2976                                 "fail to disable hotplug handling.\n");
2977                         return;
2978                 }
2979         }
2980         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2981                 if (mempools[i])
2982                         rte_mempool_free(mempools[i]);
2983         }
2984
2985         printf("\nBye...\n");
2986 }
2987
2988 typedef void (*cmd_func_t)(void);
2989 struct pmd_test_command {
2990         const char *cmd_name;
2991         cmd_func_t cmd_func;
2992 };
2993
2994 /* Check the link status of all ports in up to 9s, and print them finally */
2995 static void
2996 check_all_ports_link_status(uint32_t port_mask)
2997 {
2998 #define CHECK_INTERVAL 100 /* 100ms */
2999 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3000         portid_t portid;
3001         uint8_t count, all_ports_up, print_flag = 0;
3002         struct rte_eth_link link;
3003         int ret;
3004
3005         printf("Checking link statuses...\n");
3006         fflush(stdout);
3007         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3008                 all_ports_up = 1;
3009                 RTE_ETH_FOREACH_DEV(portid) {
3010                         if ((port_mask & (1 << portid)) == 0)
3011                                 continue;
3012                         memset(&link, 0, sizeof(link));
3013                         ret = rte_eth_link_get_nowait(portid, &link);
3014                         if (ret < 0) {
3015                                 all_ports_up = 0;
3016                                 if (print_flag == 1)
3017                                         printf("Port %u link get failed: %s\n",
3018                                                 portid, rte_strerror(-ret));
3019                                 continue;
3020                         }
3021                         /* print link status if flag set */
3022                         if (print_flag == 1) {
3023                                 if (link.link_status)
3024                                         printf(
3025                                         "Port%d Link Up. speed %u Mbps- %s\n",
3026                                         portid, link.link_speed,
3027                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3028                                         ("full-duplex") : ("half-duplex"));
3029                                 else
3030                                         printf("Port %d Link Down\n", portid);
3031                                 continue;
3032                         }
3033                         /* clear all_ports_up flag if any link down */
3034                         if (link.link_status == ETH_LINK_DOWN) {
3035                                 all_ports_up = 0;
3036                                 break;
3037                         }
3038                 }
3039                 /* after finally printing all link status, get out */
3040                 if (print_flag == 1)
3041                         break;
3042
3043                 if (all_ports_up == 0) {
3044                         fflush(stdout);
3045                         rte_delay_ms(CHECK_INTERVAL);
3046                 }
3047
3048                 /* set the print_flag if all ports up or timeout */
3049                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3050                         print_flag = 1;
3051                 }
3052
3053                 if (lsc_interrupt)
3054                         break;
3055         }
3056 }
3057
3058 /*
3059  * This callback is for remove a port for a device. It has limitation because
3060  * it is not for multiple port removal for a device.
3061  * TODO: the device detach invoke will plan to be removed from user side to
3062  * eal. And convert all PMDs to free port resources on ether device closing.
3063  */
3064 static void
3065 rmv_port_callback(void *arg)
3066 {
3067         int need_to_start = 0;
3068         int org_no_link_check = no_link_check;
3069         portid_t port_id = (intptr_t)arg;
3070         struct rte_device *dev;
3071
3072         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3073
3074         if (!test_done && port_is_forwarding(port_id)) {
3075                 need_to_start = 1;
3076                 stop_packet_forwarding();
3077         }
3078         no_link_check = 1;
3079         stop_port(port_id);
3080         no_link_check = org_no_link_check;
3081
3082         /* Save rte_device pointer before closing ethdev port */
3083         dev = rte_eth_devices[port_id].device;
3084         close_port(port_id);
3085         detach_device(dev); /* might be already removed or have more ports */
3086
3087         if (need_to_start)
3088                 start_packet_forwarding(0);
3089 }
3090
3091 /* This function is used by the interrupt thread */
3092 static int
3093 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3094                   void *ret_param)
3095 {
3096         RTE_SET_USED(param);
3097         RTE_SET_USED(ret_param);
3098
3099         if (type >= RTE_ETH_EVENT_MAX) {
3100                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3101                         port_id, __func__, type);
3102                 fflush(stderr);
3103         } else if (event_print_mask & (UINT32_C(1) << type)) {
3104                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3105                         eth_event_desc[type]);
3106                 fflush(stdout);
3107         }
3108
3109         switch (type) {
3110         case RTE_ETH_EVENT_NEW:
3111                 ports[port_id].need_setup = 1;
3112                 ports[port_id].port_status = RTE_PORT_HANDLING;
3113                 break;
3114         case RTE_ETH_EVENT_INTR_RMV:
3115                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3116                         break;
3117                 if (rte_eal_alarm_set(100000,
3118                                 rmv_port_callback, (void *)(intptr_t)port_id))
3119                         fprintf(stderr, "Could not set up deferred device removal\n");
3120                 break;
3121         default:
3122                 break;
3123         }
3124         return 0;
3125 }
3126
3127 static int
3128 register_eth_event_callback(void)
3129 {
3130         int ret;
3131         enum rte_eth_event_type event;
3132
3133         for (event = RTE_ETH_EVENT_UNKNOWN;
3134                         event < RTE_ETH_EVENT_MAX; event++) {
3135                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3136                                 event,
3137                                 eth_event_callback,
3138                                 NULL);
3139                 if (ret != 0) {
3140                         TESTPMD_LOG(ERR, "Failed to register callback for "
3141                                         "%s event\n", eth_event_desc[event]);
3142                         return -1;
3143                 }
3144         }
3145
3146         return 0;
3147 }
3148
3149 /* This function is used by the interrupt thread */
3150 static void
3151 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3152                              __rte_unused void *arg)
3153 {
3154         uint16_t port_id;
3155         int ret;
3156
3157         if (type >= RTE_DEV_EVENT_MAX) {
3158                 fprintf(stderr, "%s called upon invalid event %d\n",
3159                         __func__, type);
3160                 fflush(stderr);
3161         }
3162
3163         switch (type) {
3164         case RTE_DEV_EVENT_REMOVE:
3165                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3166                         device_name);
3167                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3168                 if (ret) {
3169                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3170                                 device_name);
3171                         return;
3172                 }
3173                 /*
3174                  * Because the user's callback is invoked in eal interrupt
3175                  * callback, the interrupt callback need to be finished before
3176                  * it can be unregistered when detaching device. So finish
3177                  * callback soon and use a deferred removal to detach device
3178                  * is need. It is a workaround, once the device detaching be
3179                  * moved into the eal in the future, the deferred removal could
3180                  * be deleted.
3181                  */
3182                 if (rte_eal_alarm_set(100000,
3183                                 rmv_port_callback, (void *)(intptr_t)port_id))
3184                         RTE_LOG(ERR, EAL,
3185                                 "Could not set up deferred device removal\n");
3186                 break;
3187         case RTE_DEV_EVENT_ADD:
3188                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3189                         device_name);
3190                 /* TODO: After finish kernel driver binding,
3191                  * begin to attach port.
3192                  */
3193                 break;
3194         default:
3195                 break;
3196         }
3197 }
3198
3199 static int
3200 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3201 {
3202         uint16_t i;
3203         int diag;
3204         uint8_t mapping_found = 0;
3205
3206         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3207                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3208                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3209                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3210                                         tx_queue_stats_mappings[i].queue_id,
3211                                         tx_queue_stats_mappings[i].stats_counter_id);
3212                         if (diag != 0)
3213                                 return diag;
3214                         mapping_found = 1;
3215                 }
3216         }
3217         if (mapping_found)
3218                 port->tx_queue_stats_mapping_enabled = 1;
3219         return 0;
3220 }
3221
3222 static int
3223 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3224 {
3225         uint16_t i;
3226         int diag;
3227         uint8_t mapping_found = 0;
3228
3229         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3230                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3231                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3232                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3233                                         rx_queue_stats_mappings[i].queue_id,
3234                                         rx_queue_stats_mappings[i].stats_counter_id);
3235                         if (diag != 0)
3236                                 return diag;
3237                         mapping_found = 1;
3238                 }
3239         }
3240         if (mapping_found)
3241                 port->rx_queue_stats_mapping_enabled = 1;
3242         return 0;
3243 }
3244
3245 static void
3246 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3247 {
3248         int diag = 0;
3249
3250         diag = set_tx_queue_stats_mapping_registers(pi, port);
3251         if (diag != 0) {
3252                 if (diag == -ENOTSUP) {
3253                         port->tx_queue_stats_mapping_enabled = 0;
3254                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3255                 }
3256                 else
3257                         rte_exit(EXIT_FAILURE,
3258                                         "set_tx_queue_stats_mapping_registers "
3259                                         "failed for port id=%d diag=%d\n",
3260                                         pi, diag);
3261         }
3262
3263         diag = set_rx_queue_stats_mapping_registers(pi, port);
3264         if (diag != 0) {
3265                 if (diag == -ENOTSUP) {
3266                         port->rx_queue_stats_mapping_enabled = 0;
3267                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3268                 }
3269                 else
3270                         rte_exit(EXIT_FAILURE,
3271                                         "set_rx_queue_stats_mapping_registers "
3272                                         "failed for port id=%d diag=%d\n",
3273                                         pi, diag);
3274         }
3275 }
3276
3277 static void
3278 rxtx_port_config(struct rte_port *port)
3279 {
3280         uint16_t qid;
3281         uint64_t offloads;
3282
3283         for (qid = 0; qid < nb_rxq; qid++) {
3284                 offloads = port->rx_conf[qid].offloads;
3285                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3286                 if (offloads != 0)
3287                         port->rx_conf[qid].offloads = offloads;
3288
3289                 /* Check if any Rx parameters have been passed */
3290                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3291                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3292
3293                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3294                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3295
3296                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3297                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3298
3299                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3300                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3301
3302                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3303                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3304
3305                 port->nb_rx_desc[qid] = nb_rxd;
3306         }
3307
3308         for (qid = 0; qid < nb_txq; qid++) {
3309                 offloads = port->tx_conf[qid].offloads;
3310                 port->tx_conf[qid] = port->dev_info.default_txconf;
3311                 if (offloads != 0)
3312                         port->tx_conf[qid].offloads = offloads;
3313
3314                 /* Check if any Tx parameters have been passed */
3315                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3316                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3317
3318                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3319                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3320
3321                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3322                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3323
3324                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3325                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3326
3327                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3328                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3329
3330                 port->nb_tx_desc[qid] = nb_txd;
3331         }
3332 }
3333
3334 void
3335 init_port_config(void)
3336 {
3337         portid_t pid;
3338         struct rte_port *port;
3339         int ret;
3340
3341         RTE_ETH_FOREACH_DEV(pid) {
3342                 port = &ports[pid];
3343                 port->dev_conf.fdir_conf = fdir_conf;
3344
3345                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3346                 if (ret != 0)
3347                         return;
3348
3349                 if (nb_rxq > 1) {
3350                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3351                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3352                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3353                 } else {
3354                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3355                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3356                 }
3357
3358                 if (port->dcb_flag == 0) {
3359                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3360                                 port->dev_conf.rxmode.mq_mode =
3361                                         (enum rte_eth_rx_mq_mode)
3362                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3363                         else
3364                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3365                 }
3366
3367                 rxtx_port_config(port);
3368
3369                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3370                 if (ret != 0)
3371                         return;
3372
3373                 map_port_queue_stats_mapping_registers(pid, port);
3374 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3375                 rte_pmd_ixgbe_bypass_init(pid);
3376 #endif
3377
3378                 if (lsc_interrupt &&
3379                     (rte_eth_devices[pid].data->dev_flags &
3380                      RTE_ETH_DEV_INTR_LSC))
3381                         port->dev_conf.intr_conf.lsc = 1;
3382                 if (rmv_interrupt &&
3383                     (rte_eth_devices[pid].data->dev_flags &
3384                      RTE_ETH_DEV_INTR_RMV))
3385                         port->dev_conf.intr_conf.rmv = 1;
3386         }
3387 }
3388
3389 void set_port_slave_flag(portid_t slave_pid)
3390 {
3391         struct rte_port *port;
3392
3393         port = &ports[slave_pid];
3394         port->slave_flag = 1;
3395 }
3396
3397 void clear_port_slave_flag(portid_t slave_pid)
3398 {
3399         struct rte_port *port;
3400
3401         port = &ports[slave_pid];
3402         port->slave_flag = 0;
3403 }
3404
3405 uint8_t port_is_bonding_slave(portid_t slave_pid)
3406 {
3407         struct rte_port *port;
3408
3409         port = &ports[slave_pid];
3410         if ((rte_eth_devices[slave_pid].data->dev_flags &
3411             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3412                 return 1;
3413         return 0;
3414 }
3415
3416 const uint16_t vlan_tags[] = {
3417                 0,  1,  2,  3,  4,  5,  6,  7,
3418                 8,  9, 10, 11,  12, 13, 14, 15,
3419                 16, 17, 18, 19, 20, 21, 22, 23,
3420                 24, 25, 26, 27, 28, 29, 30, 31
3421 };
3422
3423 static  int
3424 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3425                  enum dcb_mode_enable dcb_mode,
3426                  enum rte_eth_nb_tcs num_tcs,
3427                  uint8_t pfc_en)
3428 {
3429         uint8_t i;
3430         int32_t rc;
3431         struct rte_eth_rss_conf rss_conf;
3432
3433         /*
3434          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3435          * given above, and the number of traffic classes available for use.
3436          */
3437         if (dcb_mode == DCB_VT_ENABLED) {
3438                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3439                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3440                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3441                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3442
3443                 /* VMDQ+DCB RX and TX configurations */
3444                 vmdq_rx_conf->enable_default_pool = 0;
3445                 vmdq_rx_conf->default_pool = 0;
3446                 vmdq_rx_conf->nb_queue_pools =
3447                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3448                 vmdq_tx_conf->nb_queue_pools =
3449                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3450
3451                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3452                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3453                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3454                         vmdq_rx_conf->pool_map[i].pools =
3455                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3456                 }
3457                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3458                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3459                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3460                 }
3461
3462                 /* set DCB mode of RX and TX of multiple queues */
3463                 eth_conf->rxmode.mq_mode =
3464                                 (enum rte_eth_rx_mq_mode)
3465                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3466                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3467         } else {
3468                 struct rte_eth_dcb_rx_conf *rx_conf =
3469                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3470                 struct rte_eth_dcb_tx_conf *tx_conf =
3471                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3472
3473                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3474
3475                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3476                 if (rc != 0)
3477                         return rc;
3478
3479                 rx_conf->nb_tcs = num_tcs;
3480                 tx_conf->nb_tcs = num_tcs;
3481
3482                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3483                         rx_conf->dcb_tc[i] = i % num_tcs;
3484                         tx_conf->dcb_tc[i] = i % num_tcs;
3485                 }
3486
3487                 eth_conf->rxmode.mq_mode =
3488                                 (enum rte_eth_rx_mq_mode)
3489                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3490                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3491                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3492         }
3493
3494         if (pfc_en)
3495                 eth_conf->dcb_capability_en =
3496                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3497         else
3498                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3499
3500         return 0;
3501 }
3502
3503 int
3504 init_port_dcb_config(portid_t pid,
3505                      enum dcb_mode_enable dcb_mode,
3506                      enum rte_eth_nb_tcs num_tcs,
3507                      uint8_t pfc_en)
3508 {
3509         struct rte_eth_conf port_conf;
3510         struct rte_port *rte_port;
3511         int retval;
3512         uint16_t i;
3513
3514         rte_port = &ports[pid];
3515
3516         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3517         /* Enter DCB configuration status */
3518         dcb_config = 1;
3519
3520         port_conf.rxmode = rte_port->dev_conf.rxmode;
3521         port_conf.txmode = rte_port->dev_conf.txmode;
3522
3523         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3524         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3525         if (retval < 0)
3526                 return retval;
3527         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3528
3529         /* re-configure the device . */
3530         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3531         if (retval < 0)
3532                 return retval;
3533
3534         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3535         if (retval != 0)
3536                 return retval;
3537
3538         /* If dev_info.vmdq_pool_base is greater than 0,
3539          * the queue id of vmdq pools is started after pf queues.
3540          */
3541         if (dcb_mode == DCB_VT_ENABLED &&
3542             rte_port->dev_info.vmdq_pool_base > 0) {
3543                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3544                         " for port %d.", pid);
3545                 return -1;
3546         }
3547
3548         /* Assume the ports in testpmd have the same dcb capability
3549          * and has the same number of rxq and txq in dcb mode
3550          */
3551         if (dcb_mode == DCB_VT_ENABLED) {
3552                 if (rte_port->dev_info.max_vfs > 0) {
3553                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3554                         nb_txq = rte_port->dev_info.nb_tx_queues;
3555                 } else {
3556                         nb_rxq = rte_port->dev_info.max_rx_queues;
3557                         nb_txq = rte_port->dev_info.max_tx_queues;
3558                 }
3559         } else {
3560                 /*if vt is disabled, use all pf queues */
3561                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3562                         nb_rxq = rte_port->dev_info.max_rx_queues;
3563                         nb_txq = rte_port->dev_info.max_tx_queues;
3564                 } else {
3565                         nb_rxq = (queueid_t)num_tcs;
3566                         nb_txq = (queueid_t)num_tcs;
3567
3568                 }
3569         }
3570         rx_free_thresh = 64;
3571
3572         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3573
3574         rxtx_port_config(rte_port);
3575         /* VLAN filter */
3576         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3577         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3578                 rx_vft_set(pid, vlan_tags[i], 1);
3579
3580         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3581         if (retval != 0)
3582                 return retval;
3583
3584         map_port_queue_stats_mapping_registers(pid, rte_port);
3585
3586         rte_port->dcb_flag = 1;
3587
3588         return 0;
3589 }
3590
3591 static void
3592 init_port(void)
3593 {
3594         /* Configuration of Ethernet ports. */
3595         ports = rte_zmalloc("testpmd: ports",
3596                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3597                             RTE_CACHE_LINE_SIZE);
3598         if (ports == NULL) {
3599                 rte_exit(EXIT_FAILURE,
3600                                 "rte_zmalloc(%d struct rte_port) failed\n",
3601                                 RTE_MAX_ETHPORTS);
3602         }
3603
3604         /* Initialize ports NUMA structures */
3605         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3606         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3607         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3608 }
3609
3610 static void
3611 force_quit(void)
3612 {
3613         pmd_test_exit();
3614         prompt_exit();
3615 }
3616
3617 static void
3618 print_stats(void)
3619 {
3620         uint8_t i;
3621         const char clr[] = { 27, '[', '2', 'J', '\0' };
3622         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3623
3624         /* Clear screen and move to top left */
3625         printf("%s%s", clr, top_left);
3626
3627         printf("\nPort statistics ====================================");
3628         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3629                 nic_stats_display(fwd_ports_ids[i]);
3630
3631         fflush(stdout);
3632 }
3633
3634 static void
3635 signal_handler(int signum)
3636 {
3637         if (signum == SIGINT || signum == SIGTERM) {
3638                 printf("\nSignal %d received, preparing to exit...\n",
3639                                 signum);
3640 #ifdef RTE_LIBRTE_PDUMP
3641                 /* uninitialize packet capture framework */
3642                 rte_pdump_uninit();
3643 #endif
3644 #ifdef RTE_LIBRTE_LATENCY_STATS
3645                 if (latencystats_enabled != 0)
3646                         rte_latencystats_uninit();
3647 #endif
3648                 force_quit();
3649                 /* Set flag to indicate the force termination. */
3650                 f_quit = 1;
3651                 /* exit with the expected status */
3652                 signal(signum, SIG_DFL);
3653                 kill(getpid(), signum);
3654         }
3655 }
3656
3657 int
3658 main(int argc, char** argv)
3659 {
3660         int diag;
3661         portid_t port_id;
3662         uint16_t count;
3663         int ret;
3664
3665         signal(SIGINT, signal_handler);
3666         signal(SIGTERM, signal_handler);
3667
3668         testpmd_logtype = rte_log_register("testpmd");
3669         if (testpmd_logtype < 0)
3670                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3671         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3672
3673         diag = rte_eal_init(argc, argv);
3674         if (diag < 0)
3675                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3676                          rte_strerror(rte_errno));
3677
3678         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3679                 rte_exit(EXIT_FAILURE,
3680                          "Secondary process type not supported.\n");
3681
3682         ret = register_eth_event_callback();
3683         if (ret != 0)
3684                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3685
3686 #ifdef RTE_LIBRTE_PDUMP
3687         /* initialize packet capture framework */
3688         rte_pdump_init();
3689 #endif
3690
3691         count = 0;
3692         RTE_ETH_FOREACH_DEV(port_id) {
3693                 ports_ids[count] = port_id;
3694                 count++;
3695         }
3696         nb_ports = (portid_t) count;
3697         if (nb_ports == 0)
3698                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3699
3700         /* allocate port structures, and init them */
3701         init_port();
3702
3703         set_def_fwd_config();
3704         if (nb_lcores == 0)
3705                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3706                          "Check the core mask argument\n");
3707
3708         /* Bitrate/latency stats disabled by default */
3709 #ifdef RTE_LIBRTE_BITRATE
3710         bitrate_enabled = 0;
3711 #endif
3712 #ifdef RTE_LIBRTE_LATENCY_STATS
3713         latencystats_enabled = 0;
3714 #endif
3715
3716         /* on FreeBSD, mlockall() is disabled by default */
3717 #ifdef RTE_EXEC_ENV_FREEBSD
3718         do_mlockall = 0;
3719 #else
3720         do_mlockall = 1;
3721 #endif
3722
3723         argc -= diag;
3724         argv += diag;
3725         if (argc > 1)
3726                 launch_args_parse(argc, argv);
3727
3728         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3729                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3730                         strerror(errno));
3731         }
3732
3733         if (tx_first && interactive)
3734                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3735                                 "interactive mode.\n");
3736
3737         if (tx_first && lsc_interrupt) {
3738                 printf("Warning: lsc_interrupt needs to be off when "
3739                                 " using tx_first. Disabling.\n");
3740                 lsc_interrupt = 0;
3741         }
3742
3743         if (!nb_rxq && !nb_txq)
3744                 printf("Warning: Either rx or tx queues should be non-zero\n");
3745
3746         if (nb_rxq > 1 && nb_rxq > nb_txq)
3747                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3748                        "but nb_txq=%d will prevent to fully test it.\n",
3749                        nb_rxq, nb_txq);
3750
3751         init_config();
3752
3753         if (hot_plug) {
3754                 ret = rte_dev_hotplug_handle_enable();
3755                 if (ret) {
3756                         RTE_LOG(ERR, EAL,
3757                                 "fail to enable hotplug handling.");
3758                         return -1;
3759                 }
3760
3761                 ret = rte_dev_event_monitor_start();
3762                 if (ret) {
3763                         RTE_LOG(ERR, EAL,
3764                                 "fail to start device event monitoring.");
3765                         return -1;
3766                 }
3767
3768                 ret = rte_dev_event_callback_register(NULL,
3769                         dev_event_callback, NULL);
3770                 if (ret) {
3771                         RTE_LOG(ERR, EAL,
3772                                 "fail  to register device event callback\n");
3773                         return -1;
3774                 }
3775         }
3776
3777         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3778                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3779
3780         /* set all ports to promiscuous mode by default */
3781         RTE_ETH_FOREACH_DEV(port_id) {
3782                 ret = rte_eth_promiscuous_enable(port_id);
3783                 if (ret != 0)
3784                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3785                                 port_id, rte_strerror(-ret));
3786         }
3787
3788         /* Init metrics library */
3789         rte_metrics_init(rte_socket_id());
3790
3791 #ifdef RTE_LIBRTE_LATENCY_STATS
3792         if (latencystats_enabled != 0) {
3793                 int ret = rte_latencystats_init(1, NULL);
3794                 if (ret)
3795                         printf("Warning: latencystats init()"
3796                                 " returned error %d\n", ret);
3797                 printf("Latencystats running on lcore %d\n",
3798                         latencystats_lcore_id);
3799         }
3800 #endif
3801
3802         /* Setup bitrate stats */
3803 #ifdef RTE_LIBRTE_BITRATE
3804         if (bitrate_enabled != 0) {
3805                 bitrate_data = rte_stats_bitrate_create();
3806                 if (bitrate_data == NULL)
3807                         rte_exit(EXIT_FAILURE,
3808                                 "Could not allocate bitrate data.\n");
3809                 rte_stats_bitrate_reg(bitrate_data);
3810         }
3811 #endif
3812
3813 #ifdef RTE_LIBRTE_CMDLINE
3814         if (strlen(cmdline_filename) != 0)
3815                 cmdline_read_from_file(cmdline_filename);
3816
3817         if (interactive == 1) {
3818                 if (auto_start) {
3819                         printf("Start automatic packet forwarding\n");
3820                         start_packet_forwarding(0);
3821                 }
3822                 prompt();
3823                 pmd_test_exit();
3824         } else
3825 #endif
3826         {
3827                 char c;
3828                 int rc;
3829
3830                 f_quit = 0;
3831
3832                 printf("No commandline core given, start packet forwarding\n");
3833                 start_packet_forwarding(tx_first);
3834                 if (stats_period != 0) {
3835                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3836                         uint64_t timer_period;
3837
3838                         /* Convert to number of cycles */
3839                         timer_period = stats_period * rte_get_timer_hz();
3840
3841                         while (f_quit == 0) {
3842                                 cur_time = rte_get_timer_cycles();
3843                                 diff_time += cur_time - prev_time;
3844
3845                                 if (diff_time >= timer_period) {
3846                                         print_stats();
3847                                         /* Reset the timer */
3848                                         diff_time = 0;
3849                                 }
3850                                 /* Sleep to avoid unnecessary checks */
3851                                 prev_time = cur_time;
3852                                 sleep(1);
3853                         }
3854                 }
3855
3856                 printf("Press enter to exit\n");
3857                 rc = read(0, &c, 1);
3858                 pmd_test_exit();
3859                 if (rc < 0)
3860                         return 1;
3861         }
3862
3863         ret = rte_eal_cleanup();
3864         if (ret != 0)
3865                 rte_exit(EXIT_FAILURE,
3866                          "EAL cleanup failed: %s\n", strerror(-ret));
3867
3868         return EXIT_SUCCESS;
3869 }