app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218
 219 /*
 220  * Configuration of packet segments used by the "txonly" processing engine.
 221  */
 222 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 223 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 224         TXONLY_DEF_PACKET_LEN,
 225 };
 226 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 227
 228 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 229 /**< Split policy for packets to TX. */
 230
 231 uint8_t txonly_multi_flow;
 232 /**< Whether multiple flows are generated in TXONLY mode. */
 233
 234 uint32_t tx_pkt_times_inter;
 235 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 236
 237 uint32_t tx_pkt_times_intra;
 238 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 239
 240 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 241 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 242
 243 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 244 uint8_t dcb_config = 0;
 245
 246 /* Whether the dcb is in testing status */
 247 uint8_t dcb_test = 0;
 248
 249 /*
 250  * Configurable number of RX/TX queues.
 251  */
 252 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 253 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 254 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 255
 256 /*
 257  * Configurable number of RX/TX ring descriptors.
 258  * Defaults are supplied by drivers via ethdev.
 259  */
 260 #define RTE_TEST_RX_DESC_DEFAULT 0
 261 #define RTE_TEST_TX_DESC_DEFAULT 0
 262 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 263 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 264
 265 #define RTE_PMD_PARAM_UNSET -1
 266 /*
 267  * Configurable values of RX and TX ring threshold registers.
 268  */
 269
 270 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 271 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 272 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 273
 274 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 275 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 276 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 277
 278 /*
 279  * Configurable value of RX free threshold.
 280  */
 281 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 282
 283 /*
 284  * Configurable value of RX drop enable.
 285  */
 286 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 287
 288 /*
 289  * Configurable value of TX free threshold.
 290  */
 291 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 292
 293 /*
 294  * Configurable value of TX RS bit threshold.
 295  */
 296 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 297
 298 /*
 299  * Configurable value of buffered packets before sending.
 300  */
 301 uint16_t noisy_tx_sw_bufsz;
 302
 303 /*
 304  * Configurable value of packet buffer timeout.
 305  */
 306 uint16_t noisy_tx_sw_buf_flush_time;
 307
 308 /*
 309  * Configurable value for size of VNF internal memory area
 310  * used for simulating noisy neighbour behaviour
 311  */
 312 uint64_t noisy_lkup_mem_sz;
 313
 314 /*
 315  * Configurable value of number of random writes done in
 316  * VNF simulation memory area.
 317  */
 318 uint64_t noisy_lkup_num_writes;
 319
 320 /*
 321  * Configurable value of number of random reads done in
 322  * VNF simulation memory area.
 323  */
 324 uint64_t noisy_lkup_num_reads;
 325
 326 /*
 327  * Configurable value of number of random reads/writes done in
 328  * VNF simulation memory area.
 329  */
 330 uint64_t noisy_lkup_num_reads_writes;
 331
 332 /*
 333  * Receive Side Scaling (RSS) configuration.
 334  */
 335 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 336
 337 /*
 338  * Port topology configuration
 339  */
 340 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 341
 342 /*
 343  * Avoids to flush all the RX streams before starts forwarding.
 344  */
 345 uint8_t no_flush_rx = 0; /* flush by default */
 346
 347 /*
 348  * Flow API isolated mode.
 349  */
 350 uint8_t flow_isolate_all;
 351
 352 /*
 353  * Avoids to check link status when starting/stopping a port.
 354  */
 355 uint8_t no_link_check = 0; /* check by default */
 356
 357 /*
 358  * Don't automatically start all ports in interactive mode.
 359  */
 360 uint8_t no_device_start = 0;
 361
 362 /*
 363  * Enable link status change notification
 364  */
 365 uint8_t lsc_interrupt = 1; /* enabled by default */
 366
 367 /*
 368  * Enable device removal notification.
 369  */
 370 uint8_t rmv_interrupt = 1; /* enabled by default */
 371
 372 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 373
 374 /* After attach, port setup is called on event or by iterator */
 375 bool setup_on_probe_event = true;
 376
 377 /* Clear ptypes on port initialization. */
 378 uint8_t clear_ptypes = true;
 379
 380 /* Hairpin ports configuration mode. */
 381 uint16_t hairpin_mode;
 382
 383 /* Pretty printing of ethdev events */
 384 static const char * const eth_event_desc[] = {
 385         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 386         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 387         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 388         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 389         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 390         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 391         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 392         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 393         [RTE_ETH_EVENT_NEW] = "device probed",
 394         [RTE_ETH_EVENT_DESTROY] = "device released",
 395         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 396         [RTE_ETH_EVENT_MAX] = NULL,
 397 };
 398
 399 /*
 400  * Display or mask ether events
 401  * Default to all events except VF_MBOX
 402  */
 403 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 404                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 405                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 406                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 411 /*
 412  * Decide if all memory are locked for performance.
 413  */
 414 int do_mlockall = 0;
 415
 416 /*
 417  * NIC bypass mode configuration options.
 418  */
 419
 420 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 421 /* The NIC bypass watchdog timeout. */
 422 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 423 #endif
 424
 425
 426 #ifdef RTE_LIBRTE_LATENCY_STATS
 427
 428 /*
 429  * Set when latency stats is enabled in the commandline
 430  */
 431 uint8_t latencystats_enabled;
 432
 433 /*
 434  * Lcore ID to serive latency statistics.
 435  */
 436 lcoreid_t latencystats_lcore_id = -1;
 437
 438 #endif
 439
 440 /*
 441  * Ethernet device configuration.
 442  */
 443 struct rte_eth_rxmode rx_mode = {
 444         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 445                 /**< Default maximum frame length. */
 446 };
 447
 448 struct rte_eth_txmode tx_mode = {
 449         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 450 };
 451
 452 struct rte_fdir_conf fdir_conf = {
 453         .mode = RTE_FDIR_MODE_NONE,
 454         .pballoc = RTE_FDIR_PBALLOC_64K,
 455         .status = RTE_FDIR_REPORT_STATUS,
 456         .mask = {
 457                 .vlan_tci_mask = 0xFFEF,
 458                 .ipv4_mask     = {
 459                         .src_ip = 0xFFFFFFFF,
 460                         .dst_ip = 0xFFFFFFFF,
 461                 },
 462                 .ipv6_mask     = {
 463                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 464                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 465                 },
 466                 .src_port_mask = 0xFFFF,
 467                 .dst_port_mask = 0xFFFF,
 468                 .mac_addr_byte_mask = 0xFF,
 469                 .tunnel_type_mask = 1,
 470                 .tunnel_id_mask = 0xFFFFFFFF,
 471         },
 472         .drop_queue = 127,
 473 };
 474
 475 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 476
 477 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 478 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 479
 480 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 481 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 482
 483 uint16_t nb_tx_queue_stats_mappings = 0;
 484 uint16_t nb_rx_queue_stats_mappings = 0;
 485
 486 /*
 487  * Display zero values by default for xstats
 488  */
 489 uint8_t xstats_hide_zero;
 490
 491 /*
 492  * Measure of CPU cycles disabled by default
 493  */
 494 uint8_t record_core_cycles;
 495
 496 /*
 497  * Display of RX and TX bursts disabled by default
 498  */
 499 uint8_t record_burst_stats;
 500
 501 unsigned int num_sockets = 0;
 502 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 503
 504 #ifdef RTE_LIBRTE_BITRATESTATS
 505 /* Bitrate statistics */
 506 struct rte_stats_bitrates *bitrate_data;
 507 lcoreid_t bitrate_lcore_id;
 508 uint8_t bitrate_enabled;
 509 #endif
 510
 511 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 512 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 513
 514 /*
 515  * hexadecimal bitmask of RX mq mode can be enabled.
 516  */
 517 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 518
 519 /* Forward function declarations */
 520 static void setup_attached_port(portid_t pi);
 521 static void map_port_queue_stats_mapping_registers(portid_t pi,
 522                                                    struct rte_port *port);
 523 static void check_all_ports_link_status(uint32_t port_mask);
 524 static int eth_event_callback(portid_t port_id,
 525                               enum rte_eth_event_type type,
 526                               void *param, void *ret_param);
 527 static void dev_event_callback(const char *device_name,
 528                                 enum rte_dev_event_type type,
 529                                 void *param);
 530
 531 /*
 532  * Check if all the ports are started.
 533  * If yes, return positive value. If not, return zero.
 534  */
 535 static int all_ports_started(void);
 536
 537 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 538 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 539
 540 /* Holds the registered mbuf dynamic flags names. */
 541 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 542
 543 /*
 544  * Helper function to check if socket is already discovered.
 545  * If yes, return positive value. If not, return zero.
 546  */
 547 int
 548 new_socket_id(unsigned int socket_id)
 549 {
 550         unsigned int i;
 551
 552         for (i = 0; i < num_sockets; i++) {
 553                 if (socket_ids[i] == socket_id)
 554                         return 0;
 555         }
 556         return 1;
 557 }
 558
 559 /*
 560  * Setup default configuration.
 561  */
 562 static void
 563 set_default_fwd_lcores_config(void)
 564 {
 565         unsigned int i;
 566         unsigned int nb_lc;
 567         unsigned int sock_num;
 568
 569         nb_lc = 0;
 570         for (i = 0; i < RTE_MAX_LCORE; i++) {
 571                 if (!rte_lcore_is_enabled(i))
 572                         continue;
 573                 sock_num = rte_lcore_to_socket_id(i);
 574                 if (new_socket_id(sock_num)) {
 575                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 576                                 rte_exit(EXIT_FAILURE,
 577                                          "Total sockets greater than %u\n",
 578                                          RTE_MAX_NUMA_NODES);
 579                         }
 580                         socket_ids[num_sockets++] = sock_num;
 581                 }
 582                 if (i == rte_get_master_lcore())
 583                         continue;
 584                 fwd_lcores_cpuids[nb_lc++] = i;
 585         }
 586         nb_lcores = (lcoreid_t) nb_lc;
 587         nb_cfg_lcores = nb_lcores;
 588         nb_fwd_lcores = 1;
 589 }
 590
 591 static void
 592 set_def_peer_eth_addrs(void)
 593 {
 594         portid_t i;
 595
 596         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 597                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 598                 peer_eth_addrs[i].addr_bytes[5] = i;
 599         }
 600 }
 601
 602 static void
 603 set_default_fwd_ports_config(void)
 604 {
 605         portid_t pt_id;
 606         int i = 0;
 607
 608         RTE_ETH_FOREACH_DEV(pt_id) {
 609                 fwd_ports_ids[i++] = pt_id;
 610
 611                 /* Update sockets info according to the attached device */
 612                 int socket_id = rte_eth_dev_socket_id(pt_id);
 613                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 614                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 615                                 rte_exit(EXIT_FAILURE,
 616                                          "Total sockets greater than %u\n",
 617                                          RTE_MAX_NUMA_NODES);
 618                         }
 619                         socket_ids[num_sockets++] = socket_id;
 620                 }
 621         }
 622
 623         nb_cfg_ports = nb_ports;
 624         nb_fwd_ports = nb_ports;
 625 }
 626
 627 void
 628 set_def_fwd_config(void)
 629 {
 630         set_default_fwd_lcores_config();
 631         set_def_peer_eth_addrs();
 632         set_default_fwd_ports_config();
 633 }
 634
 635 /* extremely pessimistic estimation of memory required to create a mempool */
 636 static int
 637 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 638 {
 639         unsigned int n_pages, mbuf_per_pg, leftover;
 640         uint64_t total_mem, mbuf_mem, obj_sz;
 641
 642         /* there is no good way to predict how much space the mempool will
 643          * occupy because it will allocate chunks on the fly, and some of those
 644          * will come from default DPDK memory while some will come from our
 645          * external memory, so just assume 128MB will be enough for everyone.
 646          */
 647         uint64_t hdr_mem = 128 << 20;
 648
 649         /* account for possible non-contiguousness */
 650         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 651         if (obj_sz > pgsz) {
 652                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 653                 return -1;
 654         }
 655
 656         mbuf_per_pg = pgsz / obj_sz;
 657         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 658         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 659
 660         mbuf_mem = n_pages * pgsz;
 661
 662         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 663
 664         if (total_mem > SIZE_MAX) {
 665                 TESTPMD_LOG(ERR, "Memory size too big\n");
 666                 return -1;
 667         }
 668         *out = (size_t)total_mem;
 669
 670         return 0;
 671 }
 672
 673 static int
 674 pagesz_flags(uint64_t page_sz)
 675 {
 676         /* as per mmap() manpage, all page sizes are log2 of page size
 677          * shifted by MAP_HUGE_SHIFT
 678          */
 679         int log2 = rte_log2_u64(page_sz);
 680
 681         return (log2 << HUGE_SHIFT);
 682 }
 683
 684 static void *
 685 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 686 {
 687         void *addr;
 688         int flags;
 689
 690         /* allocate anonymous hugepages */
 691         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 692         if (huge)
 693                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 694
 695         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 696         if (addr == MAP_FAILED)
 697                 return NULL;
 698
 699         return addr;
 700 }
 701
 702 struct extmem_param {
 703         void *addr;
 704         size_t len;
 705         size_t pgsz;
 706         rte_iova_t *iova_table;
 707         unsigned int iova_table_len;
 708 };
 709
 710 static int
 711 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 712                 bool huge)
 713 {
 714         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 715                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 716         unsigned int cur_page, n_pages, pgsz_idx;
 717         size_t mem_sz, cur_pgsz;
 718         rte_iova_t *iovas = NULL;
 719         void *addr;
 720         int ret;
 721
 722         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 723                 /* skip anything that is too big */
 724                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 725                         continue;
 726
 727                 cur_pgsz = pgsizes[pgsz_idx];
 728
 729                 /* if we were told not to allocate hugepages, override */
 730                 if (!huge)
 731                         cur_pgsz = sysconf(_SC_PAGESIZE);
 732
 733                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 734                 if (ret < 0) {
 735                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 736                         return -1;
 737                 }
 738
 739                 /* allocate our memory */
 740                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 741
 742                 /* if we couldn't allocate memory with a specified page size,
 743                  * that doesn't mean we can't do it with other page sizes, so
 744                  * try another one.
 745                  */
 746                 if (addr == NULL)
 747                         continue;
 748
 749                 /* store IOVA addresses for every page in this memory area */
 750                 n_pages = mem_sz / cur_pgsz;
 751
 752                 iovas = malloc(sizeof(*iovas) * n_pages);
 753
 754                 if (iovas == NULL) {
 755                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 756                         goto fail;
 757                 }
 758                 /* lock memory if it's not huge pages */
 759                 if (!huge)
 760                         mlock(addr, mem_sz);
 761
 762                 /* populate IOVA addresses */
 763                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 764                         rte_iova_t iova;
 765                         size_t offset;
 766                         void *cur;
 767
 768                         offset = cur_pgsz * cur_page;
 769                         cur = RTE_PTR_ADD(addr, offset);
 770
 771                         /* touch the page before getting its IOVA */
 772                         *(volatile char *)cur = 0;
 773
 774                         iova = rte_mem_virt2iova(cur);
 775
 776                         iovas[cur_page] = iova;
 777                 }
 778
 779                 break;
 780         }
 781         /* if we couldn't allocate anything */
 782         if (iovas == NULL)
 783                 return -1;
 784
 785         param->addr = addr;
 786         param->len = mem_sz;
 787         param->pgsz = cur_pgsz;
 788         param->iova_table = iovas;
 789         param->iova_table_len = n_pages;
 790
 791         return 0;
 792 fail:
 793         if (iovas)
 794                 free(iovas);
 795         if (addr)
 796                 munmap(addr, mem_sz);
 797
 798         return -1;
 799 }
 800
 801 static int
 802 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 803 {
 804         struct extmem_param param;
 805         int socket_id, ret;
 806
 807         memset(&param, 0, sizeof(param));
 808
 809         /* check if our heap exists */
 810         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 811         if (socket_id < 0) {
 812                 /* create our heap */
 813                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 814                 if (ret < 0) {
 815                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 816                         return -1;
 817                 }
 818         }
 819
 820         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 821         if (ret < 0) {
 822                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 823                 return -1;
 824         }
 825
 826         /* we now have a valid memory area, so add it to heap */
 827         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 828                         param.addr, param.len, param.iova_table,
 829                         param.iova_table_len, param.pgsz);
 830
 831         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 832
 833         /* not needed any more */
 834         free(param.iova_table);
 835
 836         if (ret < 0) {
 837                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 838                 munmap(param.addr, param.len);
 839                 return -1;
 840         }
 841
 842         /* success */
 843
 844         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 845                         param.len >> 20);
 846
 847         return 0;
 848 }
 849 static void
 850 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 851              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 852 {
 853         uint16_t pid = 0;
 854         int ret;
 855
 856         RTE_ETH_FOREACH_DEV(pid) {
 857                 struct rte_eth_dev *dev =
 858                         &rte_eth_devices[pid];
 859
 860                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 861                                         memhdr->len);
 862                 if (ret) {
 863                         TESTPMD_LOG(DEBUG,
 864                                     "unable to DMA unmap addr 0x%p "
 865                                     "for device %s\n",
 866                                     memhdr->addr, dev->data->name);
 867                 }
 868         }
 869         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 870         if (ret) {
 871                 TESTPMD_LOG(DEBUG,
 872                             "unable to un-register addr 0x%p\n", memhdr->addr);
 873         }
 874 }
 875
 876 static void
 877 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 878            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 879 {
 880         uint16_t pid = 0;
 881         size_t page_size = sysconf(_SC_PAGESIZE);
 882         int ret;
 883
 884         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 885                                   page_size);
 886         if (ret) {
 887                 TESTPMD_LOG(DEBUG,
 888                             "unable to register addr 0x%p\n", memhdr->addr);
 889                 return;
 890         }
 891         RTE_ETH_FOREACH_DEV(pid) {
 892                 struct rte_eth_dev *dev =
 893                         &rte_eth_devices[pid];
 894
 895                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 896                                       memhdr->len);
 897                 if (ret) {
 898                         TESTPMD_LOG(DEBUG,
 899                                     "unable to DMA map addr 0x%p "
 900                                     "for device %s\n",
 901                                     memhdr->addr, dev->data->name);
 902                 }
 903         }
 904 }
 905
 906 static unsigned int
 907 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 908             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 909 {
 910         struct rte_pktmbuf_extmem *xmem;
 911         unsigned int ext_num, zone_num, elt_num;
 912         uint16_t elt_size;
 913
 914         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 915         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 916         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 917
 918         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 919         if (xmem == NULL) {
 920                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 921                                  "external buffer descriptors\n");
 922                 *ext_mem = NULL;
 923                 return 0;
 924         }
 925         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 926                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 927                 const struct rte_memzone *mz;
 928                 char mz_name[RTE_MEMZONE_NAMESIZE];
 929                 int ret;
 930
 931                 ret = snprintf(mz_name, sizeof(mz_name),
 932                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 933                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 934                         errno = ENAMETOOLONG;
 935                         ext_num = 0;
 936                         break;
 937                 }
 938                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 939                                                  socket_id,
 940                                                  RTE_MEMZONE_IOVA_CONTIG |
 941                                                  RTE_MEMZONE_1GB |
 942                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 943                                                  EXTBUF_ZONE_SIZE);
 944                 if (mz == NULL) {
 945                         /*
 946                          * The caller exits on external buffer creation
 947                          * error, so there is no need to free memzones.
 948                          */
 949                         errno = ENOMEM;
 950                         ext_num = 0;
 951                         break;
 952                 }
 953                 xseg->buf_ptr = mz->addr;
 954                 xseg->buf_iova = mz->iova;
 955                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 956                 xseg->elt_size = elt_size;
 957         }
 958         if (ext_num == 0 && xmem != NULL) {
 959                 free(xmem);
 960                 xmem = NULL;
 961         }
 962         *ext_mem = xmem;
 963         return ext_num;
 964 }
 965
 966 /*
 967  * Configuration initialisation done once at init time.
 968  */
 969 static struct rte_mempool *
 970 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 971                  unsigned int socket_id, uint16_t size_idx)
 972 {
 973         char pool_name[RTE_MEMPOOL_NAMESIZE];
 974         struct rte_mempool *rte_mp = NULL;
 975         uint32_t mb_size;
 976
 977         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 978         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 979
 980         TESTPMD_LOG(INFO,
 981                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 982                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 983
 984         switch (mp_alloc_type) {
 985         case MP_ALLOC_NATIVE:
 986                 {
 987                         /* wrapper to rte_mempool_create() */
 988                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 989                                         rte_mbuf_best_mempool_ops());
 990                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 991                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 992                         break;
 993                 }
 994         case MP_ALLOC_ANON:
 995                 {
 996                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 997                                 mb_size, (unsigned int) mb_mempool_cache,
 998                                 sizeof(struct rte_pktmbuf_pool_private),
 999                                 socket_id, mempool_flags);
1000                         if (rte_mp == NULL)
1001                                 goto err;
1002
1003                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1004                                 rte_mempool_free(rte_mp);
1005                                 rte_mp = NULL;
1006                                 goto err;
1007                         }
1008                         rte_pktmbuf_pool_init(rte_mp, NULL);
1009                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1010                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1011                         break;
1012                 }
1013         case MP_ALLOC_XMEM:
1014         case MP_ALLOC_XMEM_HUGE:
1015                 {
1016                         int heap_socket;
1017                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1018
1019                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1020                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1021
1022                         heap_socket =
1023                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1024                         if (heap_socket < 0)
1025                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1026
1027                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1028                                         rte_mbuf_best_mempool_ops());
1029                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1030                                         mb_mempool_cache, 0, mbuf_seg_size,
1031                                         heap_socket);
1032                         break;
1033                 }
1034         case MP_ALLOC_XBUF:
1035                 {
1036                         struct rte_pktmbuf_extmem *ext_mem;
1037                         unsigned int ext_num;
1038
1039                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1040                                                socket_id, pool_name, &ext_mem);
1041                         if (ext_num == 0)
1042                                 rte_exit(EXIT_FAILURE,
1043                                          "Can't create pinned data buffers\n");
1044
1045                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1046                                         rte_mbuf_best_mempool_ops());
1047                         rte_mp = rte_pktmbuf_pool_create_extbuf
1048                                         (pool_name, nb_mbuf, mb_mempool_cache,
1049                                          0, mbuf_seg_size, socket_id,
1050                                          ext_mem, ext_num);
1051                         free(ext_mem);
1052                         break;
1053                 }
1054         default:
1055                 {
1056                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1057                 }
1058         }
1059
1060 err:
1061         if (rte_mp == NULL) {
1062                 rte_exit(EXIT_FAILURE,
1063                         "Creation of mbuf pool for socket %u failed: %s\n",
1064                         socket_id, rte_strerror(rte_errno));
1065         } else if (verbose_level > 0) {
1066                 rte_mempool_dump(stdout, rte_mp);
1067         }
1068         return rte_mp;
1069 }
1070
1071 /*
1072  * Check given socket id is valid or not with NUMA mode,
1073  * if valid, return 0, else return -1
1074  */
1075 static int
1076 check_socket_id(const unsigned int socket_id)
1077 {
1078         static int warning_once = 0;
1079
1080         if (new_socket_id(socket_id)) {
1081                 if (!warning_once && numa_support)
1082                         printf("Warning: NUMA should be configured manually by"
1083                                " using --port-numa-config and"
1084                                " --ring-numa-config parameters along with"
1085                                " --numa.\n");
1086                 warning_once = 1;
1087                 return -1;
1088         }
1089         return 0;
1090 }
1091
1092 /*
1093  * Get the allowed maximum number of RX queues.
1094  * *pid return the port id which has minimal value of
1095  * max_rx_queues in all ports.
1096  */
1097 queueid_t
1098 get_allowed_max_nb_rxq(portid_t *pid)
1099 {
1100         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1101         bool max_rxq_valid = false;
1102         portid_t pi;
1103         struct rte_eth_dev_info dev_info;
1104
1105         RTE_ETH_FOREACH_DEV(pi) {
1106                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1107                         continue;
1108
1109                 max_rxq_valid = true;
1110                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1111                         allowed_max_rxq = dev_info.max_rx_queues;
1112                         *pid = pi;
1113                 }
1114         }
1115         return max_rxq_valid ? allowed_max_rxq : 0;
1116 }
1117
1118 /*
1119  * Check input rxq is valid or not.
1120  * If input rxq is not greater than any of maximum number
1121  * of RX queues of all ports, it is valid.
1122  * if valid, return 0, else return -1
1123  */
1124 int
1125 check_nb_rxq(queueid_t rxq)
1126 {
1127         queueid_t allowed_max_rxq;
1128         portid_t pid = 0;
1129
1130         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1131         if (rxq > allowed_max_rxq) {
1132                 printf("Fail: input rxq (%u) can't be greater "
1133                        "than max_rx_queues (%u) of port %u\n",
1134                        rxq,
1135                        allowed_max_rxq,
1136                        pid);
1137                 return -1;
1138         }
1139         return 0;
1140 }
1141
1142 /*
1143  * Get the allowed maximum number of TX queues.
1144  * *pid return the port id which has minimal value of
1145  * max_tx_queues in all ports.
1146  */
1147 queueid_t
1148 get_allowed_max_nb_txq(portid_t *pid)
1149 {
1150         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1151         bool max_txq_valid = false;
1152         portid_t pi;
1153         struct rte_eth_dev_info dev_info;
1154
1155         RTE_ETH_FOREACH_DEV(pi) {
1156                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1157                         continue;
1158
1159                 max_txq_valid = true;
1160                 if (dev_info.max_tx_queues < allowed_max_txq) {
1161                         allowed_max_txq = dev_info.max_tx_queues;
1162                         *pid = pi;
1163                 }
1164         }
1165         return max_txq_valid ? allowed_max_txq : 0;
1166 }
1167
1168 /*
1169  * Check input txq is valid or not.
1170  * If input txq is not greater than any of maximum number
1171  * of TX queues of all ports, it is valid.
1172  * if valid, return 0, else return -1
1173  */
1174 int
1175 check_nb_txq(queueid_t txq)
1176 {
1177         queueid_t allowed_max_txq;
1178         portid_t pid = 0;
1179
1180         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1181         if (txq > allowed_max_txq) {
1182                 printf("Fail: input txq (%u) can't be greater "
1183                        "than max_tx_queues (%u) of port %u\n",
1184                        txq,
1185                        allowed_max_txq,
1186                        pid);
1187                 return -1;
1188         }
1189         return 0;
1190 }
1191
1192 /*
1193  * Get the allowed maximum number of RXDs of every rx queue.
1194  * *pid return the port id which has minimal value of
1195  * max_rxd in all queues of all ports.
1196  */
1197 static uint16_t
1198 get_allowed_max_nb_rxd(portid_t *pid)
1199 {
1200         uint16_t allowed_max_rxd = UINT16_MAX;
1201         portid_t pi;
1202         struct rte_eth_dev_info dev_info;
1203
1204         RTE_ETH_FOREACH_DEV(pi) {
1205                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1206                         continue;
1207
1208                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1209                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1210                         *pid = pi;
1211                 }
1212         }
1213         return allowed_max_rxd;
1214 }
1215
1216 /*
1217  * Get the allowed minimal number of RXDs of every rx queue.
1218  * *pid return the port id which has minimal value of
1219  * min_rxd in all queues of all ports.
1220  */
1221 static uint16_t
1222 get_allowed_min_nb_rxd(portid_t *pid)
1223 {
1224         uint16_t allowed_min_rxd = 0;
1225         portid_t pi;
1226         struct rte_eth_dev_info dev_info;
1227
1228         RTE_ETH_FOREACH_DEV(pi) {
1229                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1230                         continue;
1231
1232                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1233                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1234                         *pid = pi;
1235                 }
1236         }
1237
1238         return allowed_min_rxd;
1239 }
1240
1241 /*
1242  * Check input rxd is valid or not.
1243  * If input rxd is not greater than any of maximum number
1244  * of RXDs of every Rx queues and is not less than any of
1245  * minimal number of RXDs of every Rx queues, it is valid.
1246  * if valid, return 0, else return -1
1247  */
1248 int
1249 check_nb_rxd(queueid_t rxd)
1250 {
1251         uint16_t allowed_max_rxd;
1252         uint16_t allowed_min_rxd;
1253         portid_t pid = 0;
1254
1255         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1256         if (rxd > allowed_max_rxd) {
1257                 printf("Fail: input rxd (%u) can't be greater "
1258                        "than max_rxds (%u) of port %u\n",
1259                        rxd,
1260                        allowed_max_rxd,
1261                        pid);
1262                 return -1;
1263         }
1264
1265         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1266         if (rxd < allowed_min_rxd) {
1267                 printf("Fail: input rxd (%u) can't be less "
1268                        "than min_rxds (%u) of port %u\n",
1269                        rxd,
1270                        allowed_min_rxd,
1271                        pid);
1272                 return -1;
1273         }
1274
1275         return 0;
1276 }
1277
1278 /*
1279  * Get the allowed maximum number of TXDs of every rx queues.
1280  * *pid return the port id which has minimal value of
1281  * max_txd in every tx queue.
1282  */
1283 static uint16_t
1284 get_allowed_max_nb_txd(portid_t *pid)
1285 {
1286         uint16_t allowed_max_txd = UINT16_MAX;
1287         portid_t pi;
1288         struct rte_eth_dev_info dev_info;
1289
1290         RTE_ETH_FOREACH_DEV(pi) {
1291                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1292                         continue;
1293
1294                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1295                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1296                         *pid = pi;
1297                 }
1298         }
1299         return allowed_max_txd;
1300 }
1301
1302 /*
1303  * Get the allowed maximum number of TXDs of every tx queues.
1304  * *pid return the port id which has minimal value of
1305  * min_txd in every tx queue.
1306  */
1307 static uint16_t
1308 get_allowed_min_nb_txd(portid_t *pid)
1309 {
1310         uint16_t allowed_min_txd = 0;
1311         portid_t pi;
1312         struct rte_eth_dev_info dev_info;
1313
1314         RTE_ETH_FOREACH_DEV(pi) {
1315                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1316                         continue;
1317
1318                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1319                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1320                         *pid = pi;
1321                 }
1322         }
1323
1324         return allowed_min_txd;
1325 }
1326
1327 /*
1328  * Check input txd is valid or not.
1329  * If input txd is not greater than any of maximum number
1330  * of TXDs of every Rx queues, it is valid.
1331  * if valid, return 0, else return -1
1332  */
1333 int
1334 check_nb_txd(queueid_t txd)
1335 {
1336         uint16_t allowed_max_txd;
1337         uint16_t allowed_min_txd;
1338         portid_t pid = 0;
1339
1340         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1341         if (txd > allowed_max_txd) {
1342                 printf("Fail: input txd (%u) can't be greater "
1343                        "than max_txds (%u) of port %u\n",
1344                        txd,
1345                        allowed_max_txd,
1346                        pid);
1347                 return -1;
1348         }
1349
1350         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1351         if (txd < allowed_min_txd) {
1352                 printf("Fail: input txd (%u) can't be less "
1353                        "than min_txds (%u) of port %u\n",
1354                        txd,
1355                        allowed_min_txd,
1356                        pid);
1357                 return -1;
1358         }
1359         return 0;
1360 }
1361
1362
1363 /*
1364  * Get the allowed maximum number of hairpin queues.
1365  * *pid return the port id which has minimal value of
1366  * max_hairpin_queues in all ports.
1367  */
1368 queueid_t
1369 get_allowed_max_nb_hairpinq(portid_t *pid)
1370 {
1371         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1372         portid_t pi;
1373         struct rte_eth_hairpin_cap cap;
1374
1375         RTE_ETH_FOREACH_DEV(pi) {
1376                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1377                         *pid = pi;
1378                         return 0;
1379                 }
1380                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1381                         allowed_max_hairpinq = cap.max_nb_queues;
1382                         *pid = pi;
1383                 }
1384         }
1385         return allowed_max_hairpinq;
1386 }
1387
1388 /*
1389  * Check input hairpin is valid or not.
1390  * If input hairpin is not greater than any of maximum number
1391  * of hairpin queues of all ports, it is valid.
1392  * if valid, return 0, else return -1
1393  */
1394 int
1395 check_nb_hairpinq(queueid_t hairpinq)
1396 {
1397         queueid_t allowed_max_hairpinq;
1398         portid_t pid = 0;
1399
1400         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1401         if (hairpinq > allowed_max_hairpinq) {
1402                 printf("Fail: input hairpin (%u) can't be greater "
1403                        "than max_hairpin_queues (%u) of port %u\n",
1404                        hairpinq, allowed_max_hairpinq, pid);
1405                 return -1;
1406         }
1407         return 0;
1408 }
1409
1410 static void
1411 init_config(void)
1412 {
1413         portid_t pid;
1414         struct rte_port *port;
1415         struct rte_mempool *mbp;
1416         unsigned int nb_mbuf_per_pool;
1417         lcoreid_t  lc_id;
1418         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1419         struct rte_gro_param gro_param;
1420         uint32_t gso_types;
1421         uint16_t data_size;
1422         bool warning = 0;
1423         int k;
1424         int ret;
1425
1426         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1427
1428         /* Configuration of logical cores. */
1429         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1430                                 sizeof(struct fwd_lcore *) * nb_lcores,
1431                                 RTE_CACHE_LINE_SIZE);
1432         if (fwd_lcores == NULL) {
1433                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1434                                                         "failed\n", nb_lcores);
1435         }
1436         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1437                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1438                                                sizeof(struct fwd_lcore),
1439                                                RTE_CACHE_LINE_SIZE);
1440                 if (fwd_lcores[lc_id] == NULL) {
1441                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1442                                                                 "failed\n");
1443                 }
1444                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1445         }
1446
1447         RTE_ETH_FOREACH_DEV(pid) {
1448                 port = &ports[pid];
1449                 /* Apply default TxRx configuration for all ports */
1450                 port->dev_conf.txmode = tx_mode;
1451                 port->dev_conf.rxmode = rx_mode;
1452
1453                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1454                 if (ret != 0)
1455                         rte_exit(EXIT_FAILURE,
1456                                  "rte_eth_dev_info_get() failed\n");
1457
1458                 if (!(port->dev_info.tx_offload_capa &
1459                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1460                         port->dev_conf.txmode.offloads &=
1461                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1462                 if (numa_support) {
1463                         if (port_numa[pid] != NUMA_NO_CONFIG)
1464                                 port_per_socket[port_numa[pid]]++;
1465                         else {
1466                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1467
1468                                 /*
1469                                  * if socket_id is invalid,
1470                                  * set to the first available socket.
1471                                  */
1472                                 if (check_socket_id(socket_id) < 0)
1473                                         socket_id = socket_ids[0];
1474                                 port_per_socket[socket_id]++;
1475                         }
1476                 }
1477
1478                 /* Apply Rx offloads configuration */
1479                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1480                         port->rx_conf[k].offloads =
1481                                 port->dev_conf.rxmode.offloads;
1482                 /* Apply Tx offloads configuration */
1483                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1484                         port->tx_conf[k].offloads =
1485                                 port->dev_conf.txmode.offloads;
1486
1487                 /* set flag to initialize port/queue */
1488                 port->need_reconfig = 1;
1489                 port->need_reconfig_queues = 1;
1490                 port->tx_metadata = 0;
1491
1492                 /* Check for maximum number of segments per MTU. Accordingly
1493                  * update the mbuf data size.
1494                  */
1495                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1496                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1497                         data_size = rx_mode.max_rx_pkt_len /
1498                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1499
1500                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1501                                                         mbuf_data_size[0]) {
1502                                 mbuf_data_size[0] = data_size +
1503                                                  RTE_PKTMBUF_HEADROOM;
1504                                 warning = 1;
1505                         }
1506                 }
1507         }
1508
1509         if (warning)
1510                 TESTPMD_LOG(WARNING,
1511                             "Configured mbuf size of the first segment %hu\n",
1512                             mbuf_data_size[0]);
1513         /*
1514          * Create pools of mbuf.
1515          * If NUMA support is disabled, create a single pool of mbuf in
1516          * socket 0 memory by default.
1517          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1518          *
1519          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1520          * nb_txd can be configured at run time.
1521          */
1522         if (param_total_num_mbufs)
1523                 nb_mbuf_per_pool = param_total_num_mbufs;
1524         else {
1525                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1526                         (nb_lcores * mb_mempool_cache) +
1527                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1528                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1529         }
1530
1531         if (numa_support) {
1532                 uint8_t i, j;
1533
1534                 for (i = 0; i < num_sockets; i++)
1535                         for (j = 0; j < mbuf_data_size_n; j++)
1536                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1537                                         mbuf_pool_create(mbuf_data_size[j],
1538                                                           nb_mbuf_per_pool,
1539                                                           socket_ids[i], j);
1540         } else {
1541                 uint8_t i;
1542
1543                 for (i = 0; i < mbuf_data_size_n; i++)
1544                         mempools[i] = mbuf_pool_create
1545                                         (mbuf_data_size[i],
1546                                          nb_mbuf_per_pool,
1547                                          socket_num == UMA_NO_CONFIG ?
1548                                          0 : socket_num, i);
1549         }
1550
1551         init_port_config();
1552
1553         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1554                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1555         /*
1556          * Records which Mbuf pool to use by each logical core, if needed.
1557          */
1558         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1559                 mbp = mbuf_pool_find(
1560                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1561
1562                 if (mbp == NULL)
1563                         mbp = mbuf_pool_find(0, 0);
1564                 fwd_lcores[lc_id]->mbp = mbp;
1565                 /* initialize GSO context */
1566                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1567                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1568                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1569                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1570                         RTE_ETHER_CRC_LEN;
1571                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1572         }
1573
1574         /* Configuration of packet forwarding streams. */
1575         if (init_fwd_streams() < 0)
1576                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1577
1578         fwd_config_setup();
1579
1580         /* create a gro context for each lcore */
1581         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1582         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1583         gro_param.max_item_per_flow = MAX_PKT_BURST;
1584         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1585                 gro_param.socket_id = rte_lcore_to_socket_id(
1586                                 fwd_lcores_cpuids[lc_id]);
1587                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1588                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1589                         rte_exit(EXIT_FAILURE,
1590                                         "rte_gro_ctx_create() failed\n");
1591                 }
1592         }
1593 }
1594
1595
1596 void
1597 reconfig(portid_t new_port_id, unsigned socket_id)
1598 {
1599         struct rte_port *port;
1600         int ret;
1601
1602         /* Reconfiguration of Ethernet ports. */
1603         port = &ports[new_port_id];
1604
1605         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1606         if (ret != 0)
1607                 return;
1608
1609         /* set flag to initialize port/queue */
1610         port->need_reconfig = 1;
1611         port->need_reconfig_queues = 1;
1612         port->socket_id = socket_id;
1613
1614         init_port_config();
1615 }
1616
1617
1618 int
1619 init_fwd_streams(void)
1620 {
1621         portid_t pid;
1622         struct rte_port *port;
1623         streamid_t sm_id, nb_fwd_streams_new;
1624         queueid_t q;
1625
1626         /* set socket id according to numa or not */
1627         RTE_ETH_FOREACH_DEV(pid) {
1628                 port = &ports[pid];
1629                 if (nb_rxq > port->dev_info.max_rx_queues) {
1630                         printf("Fail: nb_rxq(%d) is greater than "
1631                                 "max_rx_queues(%d)\n", nb_rxq,
1632                                 port->dev_info.max_rx_queues);
1633                         return -1;
1634                 }
1635                 if (nb_txq > port->dev_info.max_tx_queues) {
1636                         printf("Fail: nb_txq(%d) is greater than "
1637                                 "max_tx_queues(%d)\n", nb_txq,
1638                                 port->dev_info.max_tx_queues);
1639                         return -1;
1640                 }
1641                 if (numa_support) {
1642                         if (port_numa[pid] != NUMA_NO_CONFIG)
1643                                 port->socket_id = port_numa[pid];
1644                         else {
1645                                 port->socket_id = rte_eth_dev_socket_id(pid);
1646
1647                                 /*
1648                                  * if socket_id is invalid,
1649                                  * set to the first available socket.
1650                                  */
1651                                 if (check_socket_id(port->socket_id) < 0)
1652                                         port->socket_id = socket_ids[0];
1653                         }
1654                 }
1655                 else {
1656                         if (socket_num == UMA_NO_CONFIG)
1657                                 port->socket_id = 0;
1658                         else
1659                                 port->socket_id = socket_num;
1660                 }
1661         }
1662
1663         q = RTE_MAX(nb_rxq, nb_txq);
1664         if (q == 0) {
1665                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1666                 return -1;
1667         }
1668         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1669         if (nb_fwd_streams_new == nb_fwd_streams)
1670                 return 0;
1671         /* clear the old */
1672         if (fwd_streams != NULL) {
1673                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1674                         if (fwd_streams[sm_id] == NULL)
1675                                 continue;
1676                         rte_free(fwd_streams[sm_id]);
1677                         fwd_streams[sm_id] = NULL;
1678                 }
1679                 rte_free(fwd_streams);
1680                 fwd_streams = NULL;
1681         }
1682
1683         /* init new */
1684         nb_fwd_streams = nb_fwd_streams_new;
1685         if (nb_fwd_streams) {
1686                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1687                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1688                         RTE_CACHE_LINE_SIZE);
1689                 if (fwd_streams == NULL)
1690                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1691                                  " (struct fwd_stream *)) failed\n",
1692                                  nb_fwd_streams);
1693
1694                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1695                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1696                                 " struct fwd_stream", sizeof(struct fwd_stream),
1697                                 RTE_CACHE_LINE_SIZE);
1698                         if (fwd_streams[sm_id] == NULL)
1699                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1700                                          "(struct fwd_stream) failed\n");
1701                 }
1702         }
1703
1704         return 0;
1705 }
1706
1707 static void
1708 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1709 {
1710         uint64_t total_burst, sburst;
1711         uint64_t nb_burst;
1712         uint64_t burst_stats[4];
1713         uint16_t pktnb_stats[4];
1714         uint16_t nb_pkt;
1715         int burst_percent[4], sburstp;
1716         int i;
1717
1718         /*
1719          * First compute the total number of packet bursts and the
1720          * two highest numbers of bursts of the same number of packets.
1721          */
1722         memset(&burst_stats, 0x0, sizeof(burst_stats));
1723         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1724
1725         /* Show stats for 0 burst size always */
1726         total_burst = pbs->pkt_burst_spread[0];
1727         burst_stats[0] = pbs->pkt_burst_spread[0];
1728         pktnb_stats[0] = 0;
1729
1730         /* Find the next 2 burst sizes with highest occurrences. */
1731         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1732                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1733
1734                 if (nb_burst == 0)
1735                         continue;
1736
1737                 total_burst += nb_burst;
1738
1739                 if (nb_burst > burst_stats[1]) {
1740                         burst_stats[2] = burst_stats[1];
1741                         pktnb_stats[2] = pktnb_stats[1];
1742                         burst_stats[1] = nb_burst;
1743                         pktnb_stats[1] = nb_pkt;
1744                 } else if (nb_burst > burst_stats[2]) {
1745                         burst_stats[2] = nb_burst;
1746                         pktnb_stats[2] = nb_pkt;
1747                 }
1748         }
1749         if (total_burst == 0)
1750                 return;
1751
1752         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1753         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1754                 if (i == 3) {
1755                         printf("%d%% of other]\n", 100 - sburstp);
1756                         return;
1757                 }
1758
1759                 sburst += burst_stats[i];
1760                 if (sburst == total_burst) {
1761                         printf("%d%% of %d pkts]\n",
1762                                 100 - sburstp, (int) pktnb_stats[i]);
1763                         return;
1764                 }
1765
1766                 burst_percent[i] =
1767                         (double)burst_stats[i] / total_burst * 100;
1768                 printf("%d%% of %d pkts + ",
1769                         burst_percent[i], (int) pktnb_stats[i]);
1770                 sburstp += burst_percent[i];
1771         }
1772 }
1773
1774 static void
1775 fwd_stream_stats_display(streamid_t stream_id)
1776 {
1777         struct fwd_stream *fs;
1778         static const char *fwd_top_stats_border = "-------";
1779
1780         fs = fwd_streams[stream_id];
1781         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1782             (fs->fwd_dropped == 0))
1783                 return;
1784         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1785                "TX Port=%2d/Queue=%2d %s\n",
1786                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1787                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1788         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1789                " TX-dropped: %-14"PRIu64,
1790                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1791
1792         /* if checksum mode */
1793         if (cur_fwd_eng == &csum_fwd_engine) {
1794                 printf("  RX- bad IP checksum: %-14"PRIu64
1795                        "  Rx- bad L4 checksum: %-14"PRIu64
1796                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1797                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1798                         fs->rx_bad_outer_l4_csum);
1799         } else {
1800                 printf("\n");
1801         }
1802
1803         if (record_burst_stats) {
1804                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1805                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1806         }
1807 }
1808
1809 void
1810 fwd_stats_display(void)
1811 {
1812         static const char *fwd_stats_border = "----------------------";
1813         static const char *acc_stats_border = "+++++++++++++++";
1814         struct {
1815                 struct fwd_stream *rx_stream;
1816                 struct fwd_stream *tx_stream;
1817                 uint64_t tx_dropped;
1818                 uint64_t rx_bad_ip_csum;
1819                 uint64_t rx_bad_l4_csum;
1820                 uint64_t rx_bad_outer_l4_csum;
1821         } ports_stats[RTE_MAX_ETHPORTS];
1822         uint64_t total_rx_dropped = 0;
1823         uint64_t total_tx_dropped = 0;
1824         uint64_t total_rx_nombuf = 0;
1825         struct rte_eth_stats stats;
1826         uint64_t fwd_cycles = 0;
1827         uint64_t total_recv = 0;
1828         uint64_t total_xmit = 0;
1829         struct rte_port *port;
1830         streamid_t sm_id;
1831         portid_t pt_id;
1832         int i;
1833
1834         memset(ports_stats, 0, sizeof(ports_stats));
1835
1836         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1837                 struct fwd_stream *fs = fwd_streams[sm_id];
1838
1839                 if (cur_fwd_config.nb_fwd_streams >
1840                     cur_fwd_config.nb_fwd_ports) {
1841                         fwd_stream_stats_display(sm_id);
1842                 } else {
1843                         ports_stats[fs->tx_port].tx_stream = fs;
1844                         ports_stats[fs->rx_port].rx_stream = fs;
1845                 }
1846
1847                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1848
1849                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1850                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1851                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1852                                 fs->rx_bad_outer_l4_csum;
1853
1854                 if (record_core_cycles)
1855                         fwd_cycles += fs->core_cycles;
1856         }
1857         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1858                 uint8_t j;
1859
1860                 pt_id = fwd_ports_ids[i];
1861                 port = &ports[pt_id];
1862
1863                 rte_eth_stats_get(pt_id, &stats);
1864                 stats.ipackets -= port->stats.ipackets;
1865                 stats.opackets -= port->stats.opackets;
1866                 stats.ibytes -= port->stats.ibytes;
1867                 stats.obytes -= port->stats.obytes;
1868                 stats.imissed -= port->stats.imissed;
1869                 stats.oerrors -= port->stats.oerrors;
1870                 stats.rx_nombuf -= port->stats.rx_nombuf;
1871
1872                 total_recv += stats.ipackets;
1873                 total_xmit += stats.opackets;
1874                 total_rx_dropped += stats.imissed;
1875                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1876                 total_tx_dropped += stats.oerrors;
1877                 total_rx_nombuf  += stats.rx_nombuf;
1878
1879                 printf("\n  %s Forward statistics for port %-2d %s\n",
1880                        fwd_stats_border, pt_id, fwd_stats_border);
1881
1882                 if (!port->rx_queue_stats_mapping_enabled &&
1883                     !port->tx_queue_stats_mapping_enabled) {
1884                         printf("  RX-packets: %-14"PRIu64
1885                                " RX-dropped: %-14"PRIu64
1886                                "RX-total: %-"PRIu64"\n",
1887                                stats.ipackets, stats.imissed,
1888                                stats.ipackets + stats.imissed);
1889
1890                         if (cur_fwd_eng == &csum_fwd_engine)
1891                                 printf("  Bad-ipcsum: %-14"PRIu64
1892                                        " Bad-l4csum: %-14"PRIu64
1893                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1894                                        ports_stats[pt_id].rx_bad_ip_csum,
1895                                        ports_stats[pt_id].rx_bad_l4_csum,
1896                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1897                         if (stats.ierrors + stats.rx_nombuf > 0) {
1898                                 printf("  RX-error: %-"PRIu64"\n",
1899                                        stats.ierrors);
1900                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1901                                        stats.rx_nombuf);
1902                         }
1903
1904                         printf("  TX-packets: %-14"PRIu64
1905                                " TX-dropped: %-14"PRIu64
1906                                "TX-total: %-"PRIu64"\n",
1907                                stats.opackets, ports_stats[pt_id].tx_dropped,
1908                                stats.opackets + ports_stats[pt_id].tx_dropped);
1909                 } else {
1910                         printf("  RX-packets:             %14"PRIu64
1911                                "    RX-dropped:%14"PRIu64
1912                                "    RX-total:%14"PRIu64"\n",
1913                                stats.ipackets, stats.imissed,
1914                                stats.ipackets + stats.imissed);
1915
1916                         if (cur_fwd_eng == &csum_fwd_engine)
1917                                 printf("  Bad-ipcsum:%14"PRIu64
1918                                        "    Bad-l4csum:%14"PRIu64
1919                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1920                                        ports_stats[pt_id].rx_bad_ip_csum,
1921                                        ports_stats[pt_id].rx_bad_l4_csum,
1922                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1923                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1924                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1925                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1926                                        stats.rx_nombuf);
1927                         }
1928
1929                         printf("  TX-packets:             %14"PRIu64
1930                                "    TX-dropped:%14"PRIu64
1931                                "    TX-total:%14"PRIu64"\n",
1932                                stats.opackets, ports_stats[pt_id].tx_dropped,
1933                                stats.opackets + ports_stats[pt_id].tx_dropped);
1934                 }
1935
1936                 if (record_burst_stats) {
1937                         if (ports_stats[pt_id].rx_stream)
1938                                 pkt_burst_stats_display("RX",
1939                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1940                         if (ports_stats[pt_id].tx_stream)
1941                                 pkt_burst_stats_display("TX",
1942                                         &ports_stats[pt_id].tx_stream->tx_burst_stats);
1943                 }
1944
1945                 if (port->rx_queue_stats_mapping_enabled) {
1946                         printf("\n");
1947                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1948                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1949                                        "     RX-errors:%14"PRIu64
1950                                        "    RX-bytes:%14"PRIu64"\n",
1951                                        j, stats.q_ipackets[j],
1952                                        stats.q_errors[j], stats.q_ibytes[j]);
1953                         }
1954                         printf("\n");
1955                 }
1956                 if (port->tx_queue_stats_mapping_enabled) {
1957                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1958                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1959                                        "                                 TX-bytes:%14"
1960                                        PRIu64"\n",
1961                                        j, stats.q_opackets[j],
1962                                        stats.q_obytes[j]);
1963                         }
1964                 }
1965
1966                 printf("  %s--------------------------------%s\n",
1967                        fwd_stats_border, fwd_stats_border);
1968         }
1969
1970         printf("\n  %s Accumulated forward statistics for all ports"
1971                "%s\n",
1972                acc_stats_border, acc_stats_border);
1973         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1974                "%-"PRIu64"\n"
1975                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1976                "%-"PRIu64"\n",
1977                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1978                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1979         if (total_rx_nombuf > 0)
1980                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1981         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1982                "%s\n",
1983                acc_stats_border, acc_stats_border);
1984         if (record_core_cycles) {
1985 #define CYC_PER_MHZ 1E6
1986                 if (total_recv > 0 || total_xmit > 0) {
1987                         uint64_t total_pkts = 0;
1988                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1989                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1990                                 total_pkts = total_xmit;
1991                         else
1992                                 total_pkts = total_recv;
1993
1994                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1995                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1996                                " MHz Clock\n",
1997                                (double) fwd_cycles / total_pkts,
1998                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1999                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2000                 }
2001         }
2002 }
2003
2004 void
2005 fwd_stats_reset(void)
2006 {
2007         streamid_t sm_id;
2008         portid_t pt_id;
2009         int i;
2010
2011         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2012                 pt_id = fwd_ports_ids[i];
2013                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2014         }
2015         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2016                 struct fwd_stream *fs = fwd_streams[sm_id];
2017
2018                 fs->rx_packets = 0;
2019                 fs->tx_packets = 0;
2020                 fs->fwd_dropped = 0;
2021                 fs->rx_bad_ip_csum = 0;
2022                 fs->rx_bad_l4_csum = 0;
2023                 fs->rx_bad_outer_l4_csum = 0;
2024
2025                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2026                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2027                 fs->core_cycles = 0;
2028         }
2029 }
2030
2031 static void
2032 flush_fwd_rx_queues(void)
2033 {
2034         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2035         portid_t  rxp;
2036         portid_t port_id;
2037         queueid_t rxq;
2038         uint16_t  nb_rx;
2039         uint16_t  i;
2040         uint8_t   j;
2041         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2042         uint64_t timer_period;
2043
2044         /* convert to number of cycles */
2045         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2046
2047         for (j = 0; j < 2; j++) {
2048                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2049                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2050                                 port_id = fwd_ports_ids[rxp];
2051                                 /**
2052                                 * testpmd can stuck in the below do while loop
2053                                 * if rte_eth_rx_burst() always returns nonzero
2054                                 * packets. So timer is added to exit this loop
2055                                 * after 1sec timer expiry.
2056                                 */
2057                                 prev_tsc = rte_rdtsc();
2058                                 do {
2059                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2060                                                 pkts_burst, MAX_PKT_BURST);
2061                                         for (i = 0; i < nb_rx; i++)
2062                                                 rte_pktmbuf_free(pkts_burst[i]);
2063
2064                                         cur_tsc = rte_rdtsc();
2065                                         diff_tsc = cur_tsc - prev_tsc;
2066                                         timer_tsc += diff_tsc;
2067                                 } while ((nb_rx > 0) &&
2068                                         (timer_tsc < timer_period));
2069                                 timer_tsc = 0;
2070                         }
2071                 }
2072                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2073         }
2074 }
2075
2076 static void
2077 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2078 {
2079         struct fwd_stream **fsm;
2080         streamid_t nb_fs;
2081         streamid_t sm_id;
2082 #ifdef RTE_LIBRTE_BITRATESTATS
2083         uint64_t tics_per_1sec;
2084         uint64_t tics_datum;
2085         uint64_t tics_current;
2086         uint16_t i, cnt_ports;
2087
2088         cnt_ports = nb_ports;
2089         tics_datum = rte_rdtsc();
2090         tics_per_1sec = rte_get_timer_hz();
2091 #endif
2092         fsm = &fwd_streams[fc->stream_idx];
2093         nb_fs = fc->stream_nb;
2094         do {
2095                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2096                         (*pkt_fwd)(fsm[sm_id]);
2097 #ifdef RTE_LIBRTE_BITRATESTATS
2098                 if (bitrate_enabled != 0 &&
2099                                 bitrate_lcore_id == rte_lcore_id()) {
2100                         tics_current = rte_rdtsc();
2101                         if (tics_current - tics_datum >= tics_per_1sec) {
2102                                 /* Periodic bitrate calculation */
2103                                 for (i = 0; i < cnt_ports; i++)
2104                                         rte_stats_bitrate_calc(bitrate_data,
2105                                                 ports_ids[i]);
2106                                 tics_datum = tics_current;
2107                         }
2108                 }
2109 #endif
2110 #ifdef RTE_LIBRTE_LATENCY_STATS
2111                 if (latencystats_enabled != 0 &&
2112                                 latencystats_lcore_id == rte_lcore_id())
2113                         rte_latencystats_update();
2114 #endif
2115
2116         } while (! fc->stopped);
2117 }
2118
2119 static int
2120 start_pkt_forward_on_core(void *fwd_arg)
2121 {
2122         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2123                              cur_fwd_config.fwd_eng->packet_fwd);
2124         return 0;
2125 }
2126
2127 /*
2128  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2129  * Used to start communication flows in network loopback test configurations.
2130  */
2131 static int
2132 run_one_txonly_burst_on_core(void *fwd_arg)
2133 {
2134         struct fwd_lcore *fwd_lc;
2135         struct fwd_lcore tmp_lcore;
2136
2137         fwd_lc = (struct fwd_lcore *) fwd_arg;
2138         tmp_lcore = *fwd_lc;
2139         tmp_lcore.stopped = 1;
2140         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2141         return 0;
2142 }
2143
2144 /*
2145  * Launch packet forwarding:
2146  *     - Setup per-port forwarding context.
2147  *     - launch logical cores with their forwarding configuration.
2148  */
2149 static void
2150 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2151 {
2152         port_fwd_begin_t port_fwd_begin;
2153         unsigned int i;
2154         unsigned int lc_id;
2155         int diag;
2156
2157         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2158         if (port_fwd_begin != NULL) {
2159                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2160                         (*port_fwd_begin)(fwd_ports_ids[i]);
2161         }
2162         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2163                 lc_id = fwd_lcores_cpuids[i];
2164                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2165                         fwd_lcores[i]->stopped = 0;
2166                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2167                                                      fwd_lcores[i], lc_id);
2168                         if (diag != 0)
2169                                 printf("launch lcore %u failed - diag=%d\n",
2170                                        lc_id, diag);
2171                 }
2172         }
2173 }
2174
2175 /*
2176  * Launch packet forwarding configuration.
2177  */
2178 void
2179 start_packet_forwarding(int with_tx_first)
2180 {
2181         port_fwd_begin_t port_fwd_begin;
2182         port_fwd_end_t  port_fwd_end;
2183         struct rte_port *port;
2184         unsigned int i;
2185         portid_t   pt_id;
2186
2187         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2188                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2189
2190         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2191                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2192
2193         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2194                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2195                 (!nb_rxq || !nb_txq))
2196                 rte_exit(EXIT_FAILURE,
2197                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2198                         cur_fwd_eng->fwd_mode_name);
2199
2200         if (all_ports_started() == 0) {
2201                 printf("Not all ports were started\n");
2202                 return;
2203         }
2204         if (test_done == 0) {
2205                 printf("Packet forwarding already started\n");
2206                 return;
2207         }
2208
2209
2210         if(dcb_test) {
2211                 for (i = 0; i < nb_fwd_ports; i++) {
2212                         pt_id = fwd_ports_ids[i];
2213                         port = &ports[pt_id];
2214                         if (!port->dcb_flag) {
2215                                 printf("In DCB mode, all forwarding ports must "
2216                                        "be configured in this mode.\n");
2217                                 return;
2218                         }
2219                 }
2220                 if (nb_fwd_lcores == 1) {
2221                         printf("In DCB mode,the nb forwarding cores "
2222                                "should be larger than 1.\n");
2223                         return;
2224                 }
2225         }
2226         test_done = 0;
2227
2228         fwd_config_setup();
2229
2230         if(!no_flush_rx)
2231                 flush_fwd_rx_queues();
2232
2233         pkt_fwd_config_display(&cur_fwd_config);
2234         rxtx_config_display();
2235
2236         fwd_stats_reset();
2237         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2238                 pt_id = fwd_ports_ids[i];
2239                 port = &ports[pt_id];
2240                 map_port_queue_stats_mapping_registers(pt_id, port);
2241         }
2242         if (with_tx_first) {
2243                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2244                 if (port_fwd_begin != NULL) {
2245                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2246                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2247                 }
2248                 while (with_tx_first--) {
2249                         launch_packet_forwarding(
2250                                         run_one_txonly_burst_on_core);
2251                         rte_eal_mp_wait_lcore();
2252                 }
2253                 port_fwd_end = tx_only_engine.port_fwd_end;
2254                 if (port_fwd_end != NULL) {
2255                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2256                                 (*port_fwd_end)(fwd_ports_ids[i]);
2257                 }
2258         }
2259         launch_packet_forwarding(start_pkt_forward_on_core);
2260 }
2261
2262 void
2263 stop_packet_forwarding(void)
2264 {
2265         port_fwd_end_t port_fwd_end;
2266         lcoreid_t lc_id;
2267         portid_t pt_id;
2268         int i;
2269
2270         if (test_done) {
2271                 printf("Packet forwarding not started\n");
2272                 return;
2273         }
2274         printf("Telling cores to stop...");
2275         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2276                 fwd_lcores[lc_id]->stopped = 1;
2277         printf("\nWaiting for lcores to finish...\n");
2278         rte_eal_mp_wait_lcore();
2279         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2280         if (port_fwd_end != NULL) {
2281                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2282                         pt_id = fwd_ports_ids[i];
2283                         (*port_fwd_end)(pt_id);
2284                 }
2285         }
2286
2287         fwd_stats_display();
2288
2289         printf("\nDone.\n");
2290         test_done = 1;
2291 }
2292
2293 void
2294 dev_set_link_up(portid_t pid)
2295 {
2296         if (rte_eth_dev_set_link_up(pid) < 0)
2297                 printf("\nSet link up fail.\n");
2298 }
2299
2300 void
2301 dev_set_link_down(portid_t pid)
2302 {
2303         if (rte_eth_dev_set_link_down(pid) < 0)
2304                 printf("\nSet link down fail.\n");
2305 }
2306
2307 static int
2308 all_ports_started(void)
2309 {
2310         portid_t pi;
2311         struct rte_port *port;
2312
2313         RTE_ETH_FOREACH_DEV(pi) {
2314                 port = &ports[pi];
2315                 /* Check if there is a port which is not started */
2316                 if ((port->port_status != RTE_PORT_STARTED) &&
2317                         (port->slave_flag == 0))
2318                         return 0;
2319         }
2320
2321         /* No port is not started */
2322         return 1;
2323 }
2324
2325 int
2326 port_is_stopped(portid_t port_id)
2327 {
2328         struct rte_port *port = &ports[port_id];
2329
2330         if ((port->port_status != RTE_PORT_STOPPED) &&
2331             (port->slave_flag == 0))
2332                 return 0;
2333         return 1;
2334 }
2335
2336 int
2337 all_ports_stopped(void)
2338 {
2339         portid_t pi;
2340
2341         RTE_ETH_FOREACH_DEV(pi) {
2342                 if (!port_is_stopped(pi))
2343                         return 0;
2344         }
2345
2346         return 1;
2347 }
2348
2349 int
2350 port_is_started(portid_t port_id)
2351 {
2352         if (port_id_is_invalid(port_id, ENABLED_WARN))
2353                 return 0;
2354
2355         if (ports[port_id].port_status != RTE_PORT_STARTED)
2356                 return 0;
2357
2358         return 1;
2359 }
2360
2361 /* Configure the Rx and Tx hairpin queues for the selected port. */
2362 static int
2363 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2364 {
2365         queueid_t qi;
2366         struct rte_eth_hairpin_conf hairpin_conf = {
2367                 .peer_count = 1,
2368         };
2369         int i;
2370         int diag;
2371         struct rte_port *port = &ports[pi];
2372         uint16_t peer_rx_port = pi;
2373         uint16_t peer_tx_port = pi;
2374         uint32_t manual = 1;
2375         uint32_t tx_exp = hairpin_mode & 0x10;
2376
2377         if (!(hairpin_mode & 0xf)) {
2378                 peer_rx_port = pi;
2379                 peer_tx_port = pi;
2380                 manual = 0;
2381         } else if (hairpin_mode & 0x1) {
2382                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2383                                                        RTE_ETH_DEV_NO_OWNER);
2384                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2385                         peer_tx_port = rte_eth_find_next_owned_by(0,
2386                                                 RTE_ETH_DEV_NO_OWNER);
2387                 if (p_pi != RTE_MAX_ETHPORTS) {
2388                         peer_rx_port = p_pi;
2389                 } else {
2390                         uint16_t next_pi;
2391
2392                         /* Last port will be the peer RX port of the first. */
2393                         RTE_ETH_FOREACH_DEV(next_pi)
2394                                 peer_rx_port = next_pi;
2395                 }
2396                 manual = 1;
2397         } else if (hairpin_mode & 0x2) {
2398                 if (cnt_pi & 0x1) {
2399                         peer_rx_port = p_pi;
2400                 } else {
2401                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2402                                                 RTE_ETH_DEV_NO_OWNER);
2403                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2404                                 peer_rx_port = pi;
2405                 }
2406                 peer_tx_port = peer_rx_port;
2407                 manual = 1;
2408         }
2409
2410         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2411                 hairpin_conf.peers[0].port = peer_rx_port;
2412                 hairpin_conf.peers[0].queue = i + nb_rxq;
2413                 hairpin_conf.manual_bind = !!manual;
2414                 hairpin_conf.tx_explicit = !!tx_exp;
2415                 diag = rte_eth_tx_hairpin_queue_setup
2416                         (pi, qi, nb_txd, &hairpin_conf);
2417                 i++;
2418                 if (diag == 0)
2419                         continue;
2420
2421                 /* Fail to setup rx queue, return */
2422                 if (rte_atomic16_cmpset(&(port->port_status),
2423                                         RTE_PORT_HANDLING,
2424                                         RTE_PORT_STOPPED) == 0)
2425                         printf("Port %d can not be set back "
2426                                         "to stopped\n", pi);
2427                 printf("Fail to configure port %d hairpin "
2428                                 "queues\n", pi);
2429                 /* try to reconfigure queues next time */
2430                 port->need_reconfig_queues = 1;
2431                 return -1;
2432         }
2433         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2434                 hairpin_conf.peers[0].port = peer_tx_port;
2435                 hairpin_conf.peers[0].queue = i + nb_txq;
2436                 hairpin_conf.manual_bind = !!manual;
2437                 hairpin_conf.tx_explicit = !!tx_exp;
2438                 diag = rte_eth_rx_hairpin_queue_setup
2439                         (pi, qi, nb_rxd, &hairpin_conf);
2440                 i++;
2441                 if (diag == 0)
2442                         continue;
2443
2444                 /* Fail to setup rx queue, return */
2445                 if (rte_atomic16_cmpset(&(port->port_status),
2446                                         RTE_PORT_HANDLING,
2447                                         RTE_PORT_STOPPED) == 0)
2448                         printf("Port %d can not be set back "
2449                                         "to stopped\n", pi);
2450                 printf("Fail to configure port %d hairpin "
2451                                 "queues\n", pi);
2452                 /* try to reconfigure queues next time */
2453                 port->need_reconfig_queues = 1;
2454                 return -1;
2455         }
2456         return 0;
2457 }
2458
2459 int
2460 start_port(portid_t pid)
2461 {
2462         int diag, need_check_link_status = -1;
2463         portid_t pi;
2464         portid_t p_pi = RTE_MAX_ETHPORTS;
2465         portid_t pl[RTE_MAX_ETHPORTS];
2466         portid_t peer_pl[RTE_MAX_ETHPORTS];
2467         uint16_t cnt_pi = 0;
2468         uint16_t cfg_pi = 0;
2469         int peer_pi;
2470         queueid_t qi;
2471         struct rte_port *port;
2472         struct rte_ether_addr mac_addr;
2473         struct rte_eth_hairpin_cap cap;
2474
2475         if (port_id_is_invalid(pid, ENABLED_WARN))
2476                 return 0;
2477
2478         if(dcb_config)
2479                 dcb_test = 1;
2480         RTE_ETH_FOREACH_DEV(pi) {
2481                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2482                         continue;
2483
2484                 need_check_link_status = 0;
2485                 port = &ports[pi];
2486                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2487                                                  RTE_PORT_HANDLING) == 0) {
2488                         printf("Port %d is now not stopped\n", pi);
2489                         continue;
2490                 }
2491
2492                 if (port->need_reconfig > 0) {
2493                         port->need_reconfig = 0;
2494
2495                         if (flow_isolate_all) {
2496                                 int ret = port_flow_isolate(pi, 1);
2497                                 if (ret) {
2498                                         printf("Failed to apply isolated"
2499                                                " mode on port %d\n", pi);
2500                                         return -1;
2501                                 }
2502                         }
2503                         configure_rxtx_dump_callbacks(0);
2504                         printf("Configuring Port %d (socket %u)\n", pi,
2505                                         port->socket_id);
2506                         if (nb_hairpinq > 0 &&
2507                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2508                                 printf("Port %d doesn't support hairpin "
2509                                        "queues\n", pi);
2510                                 return -1;
2511                         }
2512                         /* configure port */
2513                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2514                                                      nb_txq + nb_hairpinq,
2515                                                      &(port->dev_conf));
2516                         if (diag != 0) {
2517                                 if (rte_atomic16_cmpset(&(port->port_status),
2518                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2519                                         printf("Port %d can not be set back "
2520                                                         "to stopped\n", pi);
2521                                 printf("Fail to configure port %d\n", pi);
2522                                 /* try to reconfigure port next time */
2523                                 port->need_reconfig = 1;
2524                                 return -1;
2525                         }
2526                 }
2527                 if (port->need_reconfig_queues > 0) {
2528                         port->need_reconfig_queues = 0;
2529                         /* setup tx queues */
2530                         for (qi = 0; qi < nb_txq; qi++) {
2531                                 if ((numa_support) &&
2532                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2533                                         diag = rte_eth_tx_queue_setup(pi, qi,
2534                                                 port->nb_tx_desc[qi],
2535                                                 txring_numa[pi],
2536                                                 &(port->tx_conf[qi]));
2537                                 else
2538                                         diag = rte_eth_tx_queue_setup(pi, qi,
2539                                                 port->nb_tx_desc[qi],
2540                                                 port->socket_id,
2541                                                 &(port->tx_conf[qi]));
2542
2543                                 if (diag == 0)
2544                                         continue;
2545
2546                                 /* Fail to setup tx queue, return */
2547                                 if (rte_atomic16_cmpset(&(port->port_status),
2548                                                         RTE_PORT_HANDLING,
2549                                                         RTE_PORT_STOPPED) == 0)
2550                                         printf("Port %d can not be set back "
2551                                                         "to stopped\n", pi);
2552                                 printf("Fail to configure port %d tx queues\n",
2553                                        pi);
2554                                 /* try to reconfigure queues next time */
2555                                 port->need_reconfig_queues = 1;
2556                                 return -1;
2557                         }
2558                         for (qi = 0; qi < nb_rxq; qi++) {
2559                                 /* setup rx queues */
2560                                 if ((numa_support) &&
2561                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2562                                         struct rte_mempool * mp =
2563                                                 mbuf_pool_find
2564                                                         (rxring_numa[pi], 0);
2565                                         if (mp == NULL) {
2566                                                 printf("Failed to setup RX queue:"
2567                                                         "No mempool allocation"
2568                                                         " on the socket %d\n",
2569                                                         rxring_numa[pi]);
2570                                                 return -1;
2571                                         }
2572
2573                                         diag = rte_eth_rx_queue_setup(pi, qi,
2574                                              port->nb_rx_desc[qi],
2575                                              rxring_numa[pi],
2576                                              &(port->rx_conf[qi]),
2577                                              mp);
2578                                 } else {
2579                                         struct rte_mempool *mp =
2580                                                 mbuf_pool_find
2581                                                         (port->socket_id, 0);
2582                                         if (mp == NULL) {
2583                                                 printf("Failed to setup RX queue:"
2584                                                         "No mempool allocation"
2585                                                         " on the socket %d\n",
2586                                                         port->socket_id);
2587                                                 return -1;
2588                                         }
2589                                         diag = rte_eth_rx_queue_setup(pi, qi,
2590                                              port->nb_rx_desc[qi],
2591                                              port->socket_id,
2592                                              &(port->rx_conf[qi]),
2593                                              mp);
2594                                 }
2595                                 if (diag == 0)
2596                                         continue;
2597
2598                                 /* Fail to setup rx queue, return */
2599                                 if (rte_atomic16_cmpset(&(port->port_status),
2600                                                         RTE_PORT_HANDLING,
2601                                                         RTE_PORT_STOPPED) == 0)
2602                                         printf("Port %d can not be set back "
2603                                                         "to stopped\n", pi);
2604                                 printf("Fail to configure port %d rx queues\n",
2605                                        pi);
2606                                 /* try to reconfigure queues next time */
2607                                 port->need_reconfig_queues = 1;
2608                                 return -1;
2609                         }
2610                         /* setup hairpin queues */
2611                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2612                                 return -1;
2613                 }
2614                 configure_rxtx_dump_callbacks(verbose_level);
2615                 if (clear_ptypes) {
2616                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2617                                         NULL, 0);
2618                         if (diag < 0)
2619                                 printf(
2620                                 "Port %d: Failed to disable Ptype parsing\n",
2621                                 pi);
2622                 }
2623
2624                 p_pi = pi;
2625                 cnt_pi++;
2626
2627                 /* start port */
2628                 if (rte_eth_dev_start(pi) < 0) {
2629                         printf("Fail to start port %d\n", pi);
2630
2631                         /* Fail to setup rx queue, return */
2632                         if (rte_atomic16_cmpset(&(port->port_status),
2633                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2634                                 printf("Port %d can not be set back to "
2635                                                         "stopped\n", pi);
2636                         continue;
2637                 }
2638
2639                 if (rte_atomic16_cmpset(&(port->port_status),
2640                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2641                         printf("Port %d can not be set into started\n", pi);
2642
2643                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2644                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2645                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2646                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2647                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2648
2649                 /* at least one port started, need checking link status */
2650                 need_check_link_status = 1;
2651
2652                 pl[cfg_pi++] = pi;
2653         }
2654
2655         if (need_check_link_status == 1 && !no_link_check)
2656                 check_all_ports_link_status(RTE_PORT_ALL);
2657         else if (need_check_link_status == 0)
2658                 printf("Please stop the ports first\n");
2659
2660         if (hairpin_mode & 0xf) {
2661                 uint16_t i;
2662                 int j;
2663
2664                 /* bind all started hairpin ports */
2665                 for (i = 0; i < cfg_pi; i++) {
2666                         pi = pl[i];
2667                         /* bind current Tx to all peer Rx */
2668                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2669                                                         RTE_MAX_ETHPORTS, 1);
2670                         if (peer_pi < 0)
2671                                 return peer_pi;
2672                         for (j = 0; j < peer_pi; j++) {
2673                                 if (!port_is_started(peer_pl[j]))
2674                                         continue;
2675                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2676                                 if (diag < 0) {
2677                                         printf("Error during binding hairpin"
2678                                                " Tx port %u to %u: %s\n",
2679                                                pi, peer_pl[j],
2680                                                rte_strerror(-diag));
2681                                         return -1;
2682                                 }
2683                         }
2684                         /* bind all peer Tx to current Rx */
2685                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2686                                                         RTE_MAX_ETHPORTS, 0);
2687                         if (peer_pi < 0)
2688                                 return peer_pi;
2689                         for (j = 0; j < peer_pi; j++) {
2690                                 if (!port_is_started(peer_pl[j]))
2691                                         continue;
2692                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2693                                 if (diag < 0) {
2694                                         printf("Error during binding hairpin"
2695                                                " Tx port %u to %u: %s\n",
2696                                                peer_pl[j], pi,
2697                                                rte_strerror(-diag));
2698                                         return -1;
2699                                 }
2700                         }
2701                 }
2702         }
2703
2704         printf("Done\n");
2705         return 0;
2706 }
2707
2708 void
2709 stop_port(portid_t pid)
2710 {
2711         portid_t pi;
2712         struct rte_port *port;
2713         int need_check_link_status = 0;
2714         portid_t peer_pl[RTE_MAX_ETHPORTS];
2715         int peer_pi;
2716
2717         if (dcb_test) {
2718                 dcb_test = 0;
2719                 dcb_config = 0;
2720         }
2721
2722         if (port_id_is_invalid(pid, ENABLED_WARN))
2723                 return;
2724
2725         printf("Stopping ports...\n");
2726
2727         RTE_ETH_FOREACH_DEV(pi) {
2728                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2729                         continue;
2730
2731                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2732                         printf("Please remove port %d from forwarding configuration.\n", pi);
2733                         continue;
2734                 }
2735
2736                 if (port_is_bonding_slave(pi)) {
2737                         printf("Please remove port %d from bonded device.\n", pi);
2738                         continue;
2739                 }
2740
2741                 port = &ports[pi];
2742                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2743                                                 RTE_PORT_HANDLING) == 0)
2744                         continue;
2745
2746                 if (hairpin_mode & 0xf) {
2747                         int j;
2748
2749                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2750                         /* unbind all peer Tx from current Rx */
2751                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2752                                                         RTE_MAX_ETHPORTS, 0);
2753                         if (peer_pi < 0)
2754                                 continue;
2755                         for (j = 0; j < peer_pi; j++) {
2756                                 if (!port_is_started(peer_pl[j]))
2757                                         continue;
2758                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2759                         }
2760                 }
2761
2762                 rte_eth_dev_stop(pi);
2763
2764                 if (rte_atomic16_cmpset(&(port->port_status),
2765                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2766                         printf("Port %d can not be set into stopped\n", pi);
2767                 need_check_link_status = 1;
2768         }
2769         if (need_check_link_status && !no_link_check)
2770                 check_all_ports_link_status(RTE_PORT_ALL);
2771
2772         printf("Done\n");
2773 }
2774
2775 static void
2776 remove_invalid_ports_in(portid_t *array, portid_t *total)
2777 {
2778         portid_t i;
2779         portid_t new_total = 0;
2780
2781         for (i = 0; i < *total; i++)
2782                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2783                         array[new_total] = array[i];
2784                         new_total++;
2785                 }
2786         *total = new_total;
2787 }
2788
2789 static void
2790 remove_invalid_ports(void)
2791 {
2792         remove_invalid_ports_in(ports_ids, &nb_ports);
2793         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2794         nb_cfg_ports = nb_fwd_ports;
2795 }
2796
2797 void
2798 close_port(portid_t pid)
2799 {
2800         portid_t pi;
2801         struct rte_port *port;
2802
2803         if (port_id_is_invalid(pid, ENABLED_WARN))
2804                 return;
2805
2806         printf("Closing ports...\n");
2807
2808         RTE_ETH_FOREACH_DEV(pi) {
2809                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2810                         continue;
2811
2812                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2813                         printf("Please remove port %d from forwarding configuration.\n", pi);
2814                         continue;
2815                 }
2816
2817                 if (port_is_bonding_slave(pi)) {
2818                         printf("Please remove port %d from bonded device.\n", pi);
2819                         continue;
2820                 }
2821
2822                 port = &ports[pi];
2823                 if (rte_atomic16_cmpset(&(port->port_status),
2824                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2825                         printf("Port %d is already closed\n", pi);
2826                         continue;
2827                 }
2828
2829                 port_flow_flush(pi);
2830                 rte_eth_dev_close(pi);
2831         }
2832
2833         remove_invalid_ports();
2834         printf("Done\n");
2835 }
2836
2837 void
2838 reset_port(portid_t pid)
2839 {
2840         int diag;
2841         portid_t pi;
2842         struct rte_port *port;
2843
2844         if (port_id_is_invalid(pid, ENABLED_WARN))
2845                 return;
2846
2847         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2848                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2849                 printf("Can not reset port(s), please stop port(s) first.\n");
2850                 return;
2851         }
2852
2853         printf("Resetting ports...\n");
2854
2855         RTE_ETH_FOREACH_DEV(pi) {
2856                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2857                         continue;
2858
2859                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2860                         printf("Please remove port %d from forwarding "
2861                                "configuration.\n", pi);
2862                         continue;
2863                 }
2864
2865                 if (port_is_bonding_slave(pi)) {
2866                         printf("Please remove port %d from bonded device.\n",
2867                                pi);
2868                         continue;
2869                 }
2870
2871                 diag = rte_eth_dev_reset(pi);
2872                 if (diag == 0) {
2873                         port = &ports[pi];
2874                         port->need_reconfig = 1;
2875                         port->need_reconfig_queues = 1;
2876                 } else {
2877                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2878                 }
2879         }
2880
2881         printf("Done\n");
2882 }
2883
2884 void
2885 attach_port(char *identifier)
2886 {
2887         portid_t pi;
2888         struct rte_dev_iterator iterator;
2889
2890         printf("Attaching a new port...\n");
2891
2892         if (identifier == NULL) {
2893                 printf("Invalid parameters are specified\n");
2894                 return;
2895         }
2896
2897         if (rte_dev_probe(identifier) < 0) {
2898                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2899                 return;
2900         }
2901
2902         /* first attach mode: event */
2903         if (setup_on_probe_event) {
2904                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2905                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2906                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2907                                         ports[pi].need_setup != 0)
2908                                 setup_attached_port(pi);
2909                 return;
2910         }
2911
2912         /* second attach mode: iterator */
2913         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2914                 /* setup ports matching the devargs used for probing */
2915                 if (port_is_forwarding(pi))
2916                         continue; /* port was already attached before */
2917                 setup_attached_port(pi);
2918         }
2919 }
2920
2921 static void
2922 setup_attached_port(portid_t pi)
2923 {
2924         unsigned int socket_id;
2925         int ret;
2926
2927         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2928         /* if socket_id is invalid, set to the first available socket. */
2929         if (check_socket_id(socket_id) < 0)
2930                 socket_id = socket_ids[0];
2931         reconfig(pi, socket_id);
2932         ret = rte_eth_promiscuous_enable(pi);
2933         if (ret != 0)
2934                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2935                         pi, rte_strerror(-ret));
2936
2937         ports_ids[nb_ports++] = pi;
2938         fwd_ports_ids[nb_fwd_ports++] = pi;
2939         nb_cfg_ports = nb_fwd_ports;
2940         ports[pi].need_setup = 0;
2941         ports[pi].port_status = RTE_PORT_STOPPED;
2942
2943         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2944         printf("Done\n");
2945 }
2946
2947 static void
2948 detach_device(struct rte_device *dev)
2949 {
2950         portid_t sibling;
2951
2952         if (dev == NULL) {
2953                 printf("Device already removed\n");
2954                 return;
2955         }
2956
2957         printf("Removing a device...\n");
2958
2959         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2960                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2961                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2962                                 printf("Port %u not stopped\n", sibling);
2963                                 return;
2964                         }
2965                         port_flow_flush(sibling);
2966                 }
2967         }
2968
2969         if (rte_dev_remove(dev) < 0) {
2970                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2971                 return;
2972         }
2973         remove_invalid_ports();
2974
2975         printf("Device is detached\n");
2976         printf("Now total ports is %d\n", nb_ports);
2977         printf("Done\n");
2978         return;
2979 }
2980
2981 void
2982 detach_port_device(portid_t port_id)
2983 {
2984         if (port_id_is_invalid(port_id, ENABLED_WARN))
2985                 return;
2986
2987         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2988                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2989                         printf("Port not stopped\n");
2990                         return;
2991                 }
2992                 printf("Port was not closed\n");
2993         }
2994
2995         detach_device(rte_eth_devices[port_id].device);
2996 }
2997
2998 void
2999 detach_devargs(char *identifier)
3000 {
3001         struct rte_dev_iterator iterator;
3002         struct rte_devargs da;
3003         portid_t port_id;
3004
3005         printf("Removing a device...\n");
3006
3007         memset(&da, 0, sizeof(da));
3008         if (rte_devargs_parsef(&da, "%s", identifier)) {
3009                 printf("cannot parse identifier\n");
3010                 if (da.args)
3011                         free(da.args);
3012                 return;
3013         }
3014
3015         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3016                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3017                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3018                                 printf("Port %u not stopped\n", port_id);
3019                                 rte_eth_iterator_cleanup(&iterator);
3020                                 return;
3021                         }
3022                         port_flow_flush(port_id);
3023                 }
3024         }
3025
3026         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3027                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3028                             da.name, da.bus->name);
3029                 return;
3030         }
3031
3032         remove_invalid_ports();
3033
3034         printf("Device %s is detached\n", identifier);
3035         printf("Now total ports is %d\n", nb_ports);
3036         printf("Done\n");
3037 }
3038
3039 void
3040 pmd_test_exit(void)
3041 {
3042         portid_t pt_id;
3043         unsigned int i;
3044         int ret;
3045
3046         if (test_done == 0)
3047                 stop_packet_forwarding();
3048
3049         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3050                 if (mempools[i]) {
3051                         if (mp_alloc_type == MP_ALLOC_ANON)
3052                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3053                                                      NULL);
3054                 }
3055         }
3056         if (ports != NULL) {
3057                 no_link_check = 1;
3058                 RTE_ETH_FOREACH_DEV(pt_id) {
3059                         printf("\nStopping port %d...\n", pt_id);
3060                         fflush(stdout);
3061                         stop_port(pt_id);
3062                 }
3063                 RTE_ETH_FOREACH_DEV(pt_id) {
3064                         printf("\nShutting down port %d...\n", pt_id);
3065                         fflush(stdout);
3066                         close_port(pt_id);
3067                 }
3068         }
3069
3070         if (hot_plug) {
3071                 ret = rte_dev_event_monitor_stop();
3072                 if (ret) {
3073                         RTE_LOG(ERR, EAL,
3074                                 "fail to stop device event monitor.");
3075                         return;
3076                 }
3077
3078                 ret = rte_dev_event_callback_unregister(NULL,
3079                         dev_event_callback, NULL);
3080                 if (ret < 0) {
3081                         RTE_LOG(ERR, EAL,
3082                                 "fail to unregister device event callback.\n");
3083                         return;
3084                 }
3085
3086                 ret = rte_dev_hotplug_handle_disable();
3087                 if (ret) {
3088                         RTE_LOG(ERR, EAL,
3089                                 "fail to disable hotplug handling.\n");
3090                         return;
3091                 }
3092         }
3093         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3094                 if (mempools[i])
3095                         rte_mempool_free(mempools[i]);
3096         }
3097
3098         printf("\nBye...\n");
3099 }
3100
3101 typedef void (*cmd_func_t)(void);
3102 struct pmd_test_command {
3103         const char *cmd_name;
3104         cmd_func_t cmd_func;
3105 };
3106
3107 /* Check the link status of all ports in up to 9s, and print them finally */
3108 static void
3109 check_all_ports_link_status(uint32_t port_mask)
3110 {
3111 #define CHECK_INTERVAL 100 /* 100ms */
3112 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3113         portid_t portid;
3114         uint8_t count, all_ports_up, print_flag = 0;
3115         struct rte_eth_link link;
3116         int ret;
3117         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3118
3119         printf("Checking link statuses...\n");
3120         fflush(stdout);
3121         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3122                 all_ports_up = 1;
3123                 RTE_ETH_FOREACH_DEV(portid) {
3124                         if ((port_mask & (1 << portid)) == 0)
3125                                 continue;
3126                         memset(&link, 0, sizeof(link));
3127                         ret = rte_eth_link_get_nowait(portid, &link);
3128                         if (ret < 0) {
3129                                 all_ports_up = 0;
3130                                 if (print_flag == 1)
3131                                         printf("Port %u link get failed: %s\n",
3132                                                 portid, rte_strerror(-ret));
3133                                 continue;
3134                         }
3135                         /* print link status if flag set */
3136                         if (print_flag == 1) {
3137                                 rte_eth_link_to_str(link_status,
3138                                         sizeof(link_status), &link);
3139                                 printf("Port %d %s\n", portid, link_status);
3140                                 continue;
3141                         }
3142                         /* clear all_ports_up flag if any link down */
3143                         if (link.link_status == ETH_LINK_DOWN) {
3144                                 all_ports_up = 0;
3145                                 break;
3146                         }
3147                 }
3148                 /* after finally printing all link status, get out */
3149                 if (print_flag == 1)
3150                         break;
3151
3152                 if (all_ports_up == 0) {
3153                         fflush(stdout);
3154                         rte_delay_ms(CHECK_INTERVAL);
3155                 }
3156
3157                 /* set the print_flag if all ports up or timeout */
3158                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3159                         print_flag = 1;
3160                 }
3161
3162                 if (lsc_interrupt)
3163                         break;
3164         }
3165 }
3166
3167 static void
3168 rmv_port_callback(void *arg)
3169 {
3170         int need_to_start = 0;
3171         int org_no_link_check = no_link_check;
3172         portid_t port_id = (intptr_t)arg;
3173         struct rte_device *dev;
3174
3175         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3176
3177         if (!test_done && port_is_forwarding(port_id)) {
3178                 need_to_start = 1;
3179                 stop_packet_forwarding();
3180         }
3181         no_link_check = 1;
3182         stop_port(port_id);
3183         no_link_check = org_no_link_check;
3184
3185         /* Save rte_device pointer before closing ethdev port */
3186         dev = rte_eth_devices[port_id].device;
3187         close_port(port_id);
3188         detach_device(dev); /* might be already removed or have more ports */
3189
3190         if (need_to_start)
3191                 start_packet_forwarding(0);
3192 }
3193
3194 /* This function is used by the interrupt thread */
3195 static int
3196 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3197                   void *ret_param)
3198 {
3199         RTE_SET_USED(param);
3200         RTE_SET_USED(ret_param);
3201
3202         if (type >= RTE_ETH_EVENT_MAX) {
3203                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3204                         port_id, __func__, type);
3205                 fflush(stderr);
3206         } else if (event_print_mask & (UINT32_C(1) << type)) {
3207                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3208                         eth_event_desc[type]);
3209                 fflush(stdout);
3210         }
3211
3212         switch (type) {
3213         case RTE_ETH_EVENT_NEW:
3214                 ports[port_id].need_setup = 1;
3215                 ports[port_id].port_status = RTE_PORT_HANDLING;
3216                 break;
3217         case RTE_ETH_EVENT_INTR_RMV:
3218                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3219                         break;
3220                 if (rte_eal_alarm_set(100000,
3221                                 rmv_port_callback, (void *)(intptr_t)port_id))
3222                         fprintf(stderr, "Could not set up deferred device removal\n");
3223                 break;
3224         case RTE_ETH_EVENT_DESTROY:
3225                 ports[port_id].port_status = RTE_PORT_CLOSED;
3226                 printf("Port %u is closed\n", port_id);
3227                 break;
3228         default:
3229                 break;
3230         }
3231         return 0;
3232 }
3233
3234 static int
3235 register_eth_event_callback(void)
3236 {
3237         int ret;
3238         enum rte_eth_event_type event;
3239
3240         for (event = RTE_ETH_EVENT_UNKNOWN;
3241                         event < RTE_ETH_EVENT_MAX; event++) {
3242                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3243                                 event,
3244                                 eth_event_callback,
3245                                 NULL);
3246                 if (ret != 0) {
3247                         TESTPMD_LOG(ERR, "Failed to register callback for "
3248                                         "%s event\n", eth_event_desc[event]);
3249                         return -1;
3250                 }
3251         }
3252
3253         return 0;
3254 }
3255
3256 /* This function is used by the interrupt thread */
3257 static void
3258 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3259                              __rte_unused void *arg)
3260 {
3261         uint16_t port_id;
3262         int ret;
3263
3264         if (type >= RTE_DEV_EVENT_MAX) {
3265                 fprintf(stderr, "%s called upon invalid event %d\n",
3266                         __func__, type);
3267                 fflush(stderr);
3268         }
3269
3270         switch (type) {
3271         case RTE_DEV_EVENT_REMOVE:
3272                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3273                         device_name);
3274                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3275                 if (ret) {
3276                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3277                                 device_name);
3278                         return;
3279                 }
3280                 /*
3281                  * Because the user's callback is invoked in eal interrupt
3282                  * callback, the interrupt callback need to be finished before
3283                  * it can be unregistered when detaching device. So finish
3284                  * callback soon and use a deferred removal to detach device
3285                  * is need. It is a workaround, once the device detaching be
3286                  * moved into the eal in the future, the deferred removal could
3287                  * be deleted.
3288                  */
3289                 if (rte_eal_alarm_set(100000,
3290                                 rmv_port_callback, (void *)(intptr_t)port_id))
3291                         RTE_LOG(ERR, EAL,
3292                                 "Could not set up deferred device removal\n");
3293                 break;
3294         case RTE_DEV_EVENT_ADD:
3295                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3296                         device_name);
3297                 /* TODO: After finish kernel driver binding,
3298                  * begin to attach port.
3299                  */
3300                 break;
3301         default:
3302                 break;
3303         }
3304 }
3305
3306 static int
3307 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3308 {
3309         uint16_t i;
3310         int diag;
3311         uint8_t mapping_found = 0;
3312
3313         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3314                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3315                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3316                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3317                                         tx_queue_stats_mappings[i].queue_id,
3318                                         tx_queue_stats_mappings[i].stats_counter_id);
3319                         if (diag != 0)
3320                                 return diag;
3321                         mapping_found = 1;
3322                 }
3323         }
3324         if (mapping_found)
3325                 port->tx_queue_stats_mapping_enabled = 1;
3326         return 0;
3327 }
3328
3329 static int
3330 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3331 {
3332         uint16_t i;
3333         int diag;
3334         uint8_t mapping_found = 0;
3335
3336         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3337                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3338                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3339                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3340                                         rx_queue_stats_mappings[i].queue_id,
3341                                         rx_queue_stats_mappings[i].stats_counter_id);
3342                         if (diag != 0)
3343                                 return diag;
3344                         mapping_found = 1;
3345                 }
3346         }
3347         if (mapping_found)
3348                 port->rx_queue_stats_mapping_enabled = 1;
3349         return 0;
3350 }
3351
3352 static void
3353 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3354 {
3355         int diag = 0;
3356
3357         diag = set_tx_queue_stats_mapping_registers(pi, port);
3358         if (diag != 0) {
3359                 if (diag == -ENOTSUP) {
3360                         port->tx_queue_stats_mapping_enabled = 0;
3361                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3362                 }
3363                 else
3364                         rte_exit(EXIT_FAILURE,
3365                                         "set_tx_queue_stats_mapping_registers "
3366                                         "failed for port id=%d diag=%d\n",
3367                                         pi, diag);
3368         }
3369
3370         diag = set_rx_queue_stats_mapping_registers(pi, port);
3371         if (diag != 0) {
3372                 if (diag == -ENOTSUP) {
3373                         port->rx_queue_stats_mapping_enabled = 0;
3374                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3375                 }
3376                 else
3377                         rte_exit(EXIT_FAILURE,
3378                                         "set_rx_queue_stats_mapping_registers "
3379                                         "failed for port id=%d diag=%d\n",
3380                                         pi, diag);
3381         }
3382 }
3383
3384 static void
3385 rxtx_port_config(struct rte_port *port)
3386 {
3387         uint16_t qid;
3388         uint64_t offloads;
3389
3390         for (qid = 0; qid < nb_rxq; qid++) {
3391                 offloads = port->rx_conf[qid].offloads;
3392                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3393                 if (offloads != 0)
3394                         port->rx_conf[qid].offloads = offloads;
3395
3396                 /* Check if any Rx parameters have been passed */
3397                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3398                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3399
3400                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3401                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3402
3403                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3404                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3405
3406                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3407                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3408
3409                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3410                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3411
3412                 port->nb_rx_desc[qid] = nb_rxd;
3413         }
3414
3415         for (qid = 0; qid < nb_txq; qid++) {
3416                 offloads = port->tx_conf[qid].offloads;
3417                 port->tx_conf[qid] = port->dev_info.default_txconf;
3418                 if (offloads != 0)
3419                         port->tx_conf[qid].offloads = offloads;
3420
3421                 /* Check if any Tx parameters have been passed */
3422                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3423                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3424
3425                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3426                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3427
3428                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3429                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3430
3431                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3432                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3433
3434                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3435                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3436
3437                 port->nb_tx_desc[qid] = nb_txd;
3438         }
3439 }
3440
3441 void
3442 init_port_config(void)
3443 {
3444         portid_t pid;
3445         struct rte_port *port;
3446         int ret;
3447
3448         RTE_ETH_FOREACH_DEV(pid) {
3449                 port = &ports[pid];
3450                 port->dev_conf.fdir_conf = fdir_conf;
3451
3452                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3453                 if (ret != 0)
3454                         return;
3455
3456                 if (nb_rxq > 1) {
3457                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3458                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3459                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3460                 } else {
3461                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3462                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3463                 }
3464
3465                 if (port->dcb_flag == 0) {
3466                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3467                                 port->dev_conf.rxmode.mq_mode =
3468                                         (enum rte_eth_rx_mq_mode)
3469                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3470                         else
3471                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3472                 }
3473
3474                 rxtx_port_config(port);
3475
3476                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3477                 if (ret != 0)
3478                         return;
3479
3480                 map_port_queue_stats_mapping_registers(pid, port);
3481 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3482                 rte_pmd_ixgbe_bypass_init(pid);
3483 #endif
3484
3485                 if (lsc_interrupt &&
3486                     (rte_eth_devices[pid].data->dev_flags &
3487                      RTE_ETH_DEV_INTR_LSC))
3488                         port->dev_conf.intr_conf.lsc = 1;
3489                 if (rmv_interrupt &&
3490                     (rte_eth_devices[pid].data->dev_flags &
3491                      RTE_ETH_DEV_INTR_RMV))
3492                         port->dev_conf.intr_conf.rmv = 1;
3493         }
3494 }
3495
3496 void set_port_slave_flag(portid_t slave_pid)
3497 {
3498         struct rte_port *port;
3499
3500         port = &ports[slave_pid];
3501         port->slave_flag = 1;
3502 }
3503
3504 void clear_port_slave_flag(portid_t slave_pid)
3505 {
3506         struct rte_port *port;
3507
3508         port = &ports[slave_pid];
3509         port->slave_flag = 0;
3510 }
3511
3512 uint8_t port_is_bonding_slave(portid_t slave_pid)
3513 {
3514         struct rte_port *port;
3515
3516         port = &ports[slave_pid];
3517         if ((rte_eth_devices[slave_pid].data->dev_flags &
3518             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3519                 return 1;
3520         return 0;
3521 }
3522
3523 const uint16_t vlan_tags[] = {
3524                 0,  1,  2,  3,  4,  5,  6,  7,
3525                 8,  9, 10, 11,  12, 13, 14, 15,
3526                 16, 17, 18, 19, 20, 21, 22, 23,
3527                 24, 25, 26, 27, 28, 29, 30, 31
3528 };
3529
3530 static  int
3531 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3532                  enum dcb_mode_enable dcb_mode,
3533                  enum rte_eth_nb_tcs num_tcs,
3534                  uint8_t pfc_en)
3535 {
3536         uint8_t i;
3537         int32_t rc;
3538         struct rte_eth_rss_conf rss_conf;
3539
3540         /*
3541          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3542          * given above, and the number of traffic classes available for use.
3543          */
3544         if (dcb_mode == DCB_VT_ENABLED) {
3545                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3546                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3547                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3548                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3549
3550                 /* VMDQ+DCB RX and TX configurations */
3551                 vmdq_rx_conf->enable_default_pool = 0;
3552                 vmdq_rx_conf->default_pool = 0;
3553                 vmdq_rx_conf->nb_queue_pools =
3554                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3555                 vmdq_tx_conf->nb_queue_pools =
3556                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3557
3558                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3559                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3560                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3561                         vmdq_rx_conf->pool_map[i].pools =
3562                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3563                 }
3564                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3565                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3566                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3567                 }
3568
3569                 /* set DCB mode of RX and TX of multiple queues */
3570                 eth_conf->rxmode.mq_mode =
3571                                 (enum rte_eth_rx_mq_mode)
3572                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3573                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3574         } else {
3575                 struct rte_eth_dcb_rx_conf *rx_conf =
3576                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3577                 struct rte_eth_dcb_tx_conf *tx_conf =
3578                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3579
3580                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3581
3582                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3583                 if (rc != 0)
3584                         return rc;
3585
3586                 rx_conf->nb_tcs = num_tcs;
3587                 tx_conf->nb_tcs = num_tcs;
3588
3589                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3590                         rx_conf->dcb_tc[i] = i % num_tcs;
3591                         tx_conf->dcb_tc[i] = i % num_tcs;
3592                 }
3593
3594                 eth_conf->rxmode.mq_mode =
3595                                 (enum rte_eth_rx_mq_mode)
3596                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3597                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3598                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3599         }
3600
3601         if (pfc_en)
3602                 eth_conf->dcb_capability_en =
3603                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3604         else
3605                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3606
3607         return 0;
3608 }
3609
3610 int
3611 init_port_dcb_config(portid_t pid,
3612                      enum dcb_mode_enable dcb_mode,
3613                      enum rte_eth_nb_tcs num_tcs,
3614                      uint8_t pfc_en)
3615 {
3616         struct rte_eth_conf port_conf;
3617         struct rte_port *rte_port;
3618         int retval;
3619         uint16_t i;
3620
3621         rte_port = &ports[pid];
3622
3623         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3624         /* Enter DCB configuration status */
3625         dcb_config = 1;
3626
3627         port_conf.rxmode = rte_port->dev_conf.rxmode;
3628         port_conf.txmode = rte_port->dev_conf.txmode;
3629
3630         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3631         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3632         if (retval < 0)
3633                 return retval;
3634         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3635
3636         /* re-configure the device . */
3637         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3638         if (retval < 0)
3639                 return retval;
3640
3641         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3642         if (retval != 0)
3643                 return retval;
3644
3645         /* If dev_info.vmdq_pool_base is greater than 0,
3646          * the queue id of vmdq pools is started after pf queues.
3647          */
3648         if (dcb_mode == DCB_VT_ENABLED &&
3649             rte_port->dev_info.vmdq_pool_base > 0) {
3650                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3651                         " for port %d.", pid);
3652                 return -1;
3653         }
3654
3655         /* Assume the ports in testpmd have the same dcb capability
3656          * and has the same number of rxq and txq in dcb mode
3657          */
3658         if (dcb_mode == DCB_VT_ENABLED) {
3659                 if (rte_port->dev_info.max_vfs > 0) {
3660                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3661                         nb_txq = rte_port->dev_info.nb_tx_queues;
3662                 } else {
3663                         nb_rxq = rte_port->dev_info.max_rx_queues;
3664                         nb_txq = rte_port->dev_info.max_tx_queues;
3665                 }
3666         } else {
3667                 /*if vt is disabled, use all pf queues */
3668                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3669                         nb_rxq = rte_port->dev_info.max_rx_queues;
3670                         nb_txq = rte_port->dev_info.max_tx_queues;
3671                 } else {
3672                         nb_rxq = (queueid_t)num_tcs;
3673                         nb_txq = (queueid_t)num_tcs;
3674
3675                 }
3676         }
3677         rx_free_thresh = 64;
3678
3679         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3680
3681         rxtx_port_config(rte_port);
3682         /* VLAN filter */
3683         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3684         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3685                 rx_vft_set(pid, vlan_tags[i], 1);
3686
3687         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3688         if (retval != 0)
3689                 return retval;
3690
3691         map_port_queue_stats_mapping_registers(pid, rte_port);
3692
3693         rte_port->dcb_flag = 1;
3694
3695         return 0;
3696 }
3697
3698 static void
3699 init_port(void)
3700 {
3701         int i;
3702
3703         /* Configuration of Ethernet ports. */
3704         ports = rte_zmalloc("testpmd: ports",
3705                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3706                             RTE_CACHE_LINE_SIZE);
3707         if (ports == NULL) {
3708                 rte_exit(EXIT_FAILURE,
3709                                 "rte_zmalloc(%d struct rte_port) failed\n",
3710                                 RTE_MAX_ETHPORTS);
3711         }
3712         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3713                 LIST_INIT(&ports[i].flow_tunnel_list);
3714         /* Initialize ports NUMA structures */
3715         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3716         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3717         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3718 }
3719
3720 static void
3721 force_quit(void)
3722 {
3723         pmd_test_exit();
3724         prompt_exit();
3725 }
3726
3727 static void
3728 print_stats(void)
3729 {
3730         uint8_t i;
3731         const char clr[] = { 27, '[', '2', 'J', '\0' };
3732         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3733
3734         /* Clear screen and move to top left */
3735         printf("%s%s", clr, top_left);
3736
3737         printf("\nPort statistics ====================================");
3738         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3739                 nic_stats_display(fwd_ports_ids[i]);
3740
3741         fflush(stdout);
3742 }
3743
3744 static void
3745 signal_handler(int signum)
3746 {
3747         if (signum == SIGINT || signum == SIGTERM) {
3748                 printf("\nSignal %d received, preparing to exit...\n",
3749                                 signum);
3750 #ifdef RTE_LIBRTE_PDUMP
3751                 /* uninitialize packet capture framework */
3752                 rte_pdump_uninit();
3753 #endif
3754 #ifdef RTE_LIBRTE_LATENCY_STATS
3755                 if (latencystats_enabled != 0)
3756                         rte_latencystats_uninit();
3757 #endif
3758                 force_quit();
3759                 /* Set flag to indicate the force termination. */
3760                 f_quit = 1;
3761                 /* exit with the expected status */
3762                 signal(signum, SIG_DFL);
3763                 kill(getpid(), signum);
3764         }
3765 }
3766
3767 int
3768 main(int argc, char** argv)
3769 {
3770         int diag;
3771         portid_t port_id;
3772         uint16_t count;
3773         int ret;
3774
3775         signal(SIGINT, signal_handler);
3776         signal(SIGTERM, signal_handler);
3777
3778         testpmd_logtype = rte_log_register("testpmd");
3779         if (testpmd_logtype < 0)
3780                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3781         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3782
3783         diag = rte_eal_init(argc, argv);
3784         if (diag < 0)
3785                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3786                          rte_strerror(rte_errno));
3787
3788         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3789                 rte_exit(EXIT_FAILURE,
3790                          "Secondary process type not supported.\n");
3791
3792         ret = register_eth_event_callback();
3793         if (ret != 0)
3794                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3795
3796 #ifdef RTE_LIBRTE_PDUMP
3797         /* initialize packet capture framework */
3798         rte_pdump_init();
3799 #endif
3800
3801         count = 0;
3802         RTE_ETH_FOREACH_DEV(port_id) {
3803                 ports_ids[count] = port_id;
3804                 count++;
3805         }
3806         nb_ports = (portid_t) count;
3807         if (nb_ports == 0)
3808                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3809
3810         /* allocate port structures, and init them */
3811         init_port();
3812
3813         set_def_fwd_config();
3814         if (nb_lcores == 0)
3815                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3816                          "Check the core mask argument\n");
3817
3818         /* Bitrate/latency stats disabled by default */
3819 #ifdef RTE_LIBRTE_BITRATESTATS
3820         bitrate_enabled = 0;
3821 #endif
3822 #ifdef RTE_LIBRTE_LATENCY_STATS
3823         latencystats_enabled = 0;
3824 #endif
3825
3826         /* on FreeBSD, mlockall() is disabled by default */
3827 #ifdef RTE_EXEC_ENV_FREEBSD
3828         do_mlockall = 0;
3829 #else
3830         do_mlockall = 1;
3831 #endif
3832
3833         argc -= diag;
3834         argv += diag;
3835         if (argc > 1)
3836                 launch_args_parse(argc, argv);
3837
3838         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3839                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3840                         strerror(errno));
3841         }
3842
3843         if (tx_first && interactive)
3844                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3845                                 "interactive mode.\n");
3846
3847         if (tx_first && lsc_interrupt) {
3848                 printf("Warning: lsc_interrupt needs to be off when "
3849                                 " using tx_first. Disabling.\n");
3850                 lsc_interrupt = 0;
3851         }
3852
3853         if (!nb_rxq && !nb_txq)
3854                 printf("Warning: Either rx or tx queues should be non-zero\n");
3855
3856         if (nb_rxq > 1 && nb_rxq > nb_txq)
3857                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3858                        "but nb_txq=%d will prevent to fully test it.\n",
3859                        nb_rxq, nb_txq);
3860
3861         init_config();
3862
3863         if (hot_plug) {
3864                 ret = rte_dev_hotplug_handle_enable();
3865                 if (ret) {
3866                         RTE_LOG(ERR, EAL,
3867                                 "fail to enable hotplug handling.");
3868                         return -1;
3869                 }
3870
3871                 ret = rte_dev_event_monitor_start();
3872                 if (ret) {
3873                         RTE_LOG(ERR, EAL,
3874                                 "fail to start device event monitoring.");
3875                         return -1;
3876                 }
3877
3878                 ret = rte_dev_event_callback_register(NULL,
3879                         dev_event_callback, NULL);
3880                 if (ret) {
3881                         RTE_LOG(ERR, EAL,
3882                                 "fail  to register device event callback\n");
3883                         return -1;
3884                 }
3885         }
3886
3887         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3888                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3889
3890         /* set all ports to promiscuous mode by default */
3891         RTE_ETH_FOREACH_DEV(port_id) {
3892                 ret = rte_eth_promiscuous_enable(port_id);
3893                 if (ret != 0)
3894                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3895                                 port_id, rte_strerror(-ret));
3896         }
3897
3898         /* Init metrics library */
3899         rte_metrics_init(rte_socket_id());
3900
3901 #ifdef RTE_LIBRTE_LATENCY_STATS
3902         if (latencystats_enabled != 0) {
3903                 int ret = rte_latencystats_init(1, NULL);
3904                 if (ret)
3905                         printf("Warning: latencystats init()"
3906                                 " returned error %d\n", ret);
3907                 printf("Latencystats running on lcore %d\n",
3908                         latencystats_lcore_id);
3909         }
3910 #endif
3911
3912         /* Setup bitrate stats */
3913 #ifdef RTE_LIBRTE_BITRATESTATS
3914         if (bitrate_enabled != 0) {
3915                 bitrate_data = rte_stats_bitrate_create();
3916                 if (bitrate_data == NULL)
3917                         rte_exit(EXIT_FAILURE,
3918                                 "Could not allocate bitrate data.\n");
3919                 rte_stats_bitrate_reg(bitrate_data);
3920         }
3921 #endif
3922
3923 #ifdef RTE_LIBRTE_CMDLINE
3924         if (strlen(cmdline_filename) != 0)
3925                 cmdline_read_from_file(cmdline_filename);
3926
3927         if (interactive == 1) {
3928                 if (auto_start) {
3929                         printf("Start automatic packet forwarding\n");
3930                         start_packet_forwarding(0);
3931                 }
3932                 prompt();
3933                 pmd_test_exit();
3934         } else
3935 #endif
3936         {
3937                 char c;
3938                 int rc;
3939
3940                 f_quit = 0;
3941
3942                 printf("No commandline core given, start packet forwarding\n");
3943                 start_packet_forwarding(tx_first);
3944                 if (stats_period != 0) {
3945                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3946                         uint64_t timer_period;
3947
3948                         /* Convert to number of cycles */
3949                         timer_period = stats_period * rte_get_timer_hz();
3950
3951                         while (f_quit == 0) {
3952                                 cur_time = rte_get_timer_cycles();
3953                                 diff_time += cur_time - prev_time;
3954
3955                                 if (diff_time >= timer_period) {
3956                                         print_stats();
3957                                         /* Reset the timer */
3958                                         diff_time = 0;
3959                                 }
3960                                 /* Sleep to avoid unnecessary checks */
3961                                 prev_time = cur_time;
3962                                 sleep(1);
3963                         }
3964                 }
3965
3966                 printf("Press enter to exit\n");
3967                 rc = read(0, &c, 1);
3968                 pmd_test_exit();
3969                 if (rc < 0)
3970                         return 1;
3971         }
3972
3973         ret = rte_eal_cleanup();
3974         if (ret != 0)
3975                 rte_exit(EXIT_FAILURE,
3976                          "EAL cleanup failed: %s\n", strerror(-ret));
3977
3978         return EXIT_SUCCESS;
3979 }