app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 239 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 240 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 241
 242 /*
 243  * Configurable number of RX/TX ring descriptors.
 244  * Defaults are supplied by drivers via ethdev.
 245  */
 246 #define RTE_TEST_RX_DESC_DEFAULT 0
 247 #define RTE_TEST_TX_DESC_DEFAULT 0
 248 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 249 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 250
 251 #define RTE_PMD_PARAM_UNSET -1
 252 /*
 253  * Configurable values of RX and TX ring threshold registers.
 254  */
 255
 256 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 258 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 259
 260 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 262 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 263
 264 /*
 265  * Configurable value of RX free threshold.
 266  */
 267 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 268
 269 /*
 270  * Configurable value of RX drop enable.
 271  */
 272 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 273
 274 /*
 275  * Configurable value of TX free threshold.
 276  */
 277 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 278
 279 /*
 280  * Configurable value of TX RS bit threshold.
 281  */
 282 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 283
 284 /*
 285  * Configurable value of buffered packets before sending.
 286  */
 287 uint16_t noisy_tx_sw_bufsz;
 288
 289 /*
 290  * Configurable value of packet buffer timeout.
 291  */
 292 uint16_t noisy_tx_sw_buf_flush_time;
 293
 294 /*
 295  * Configurable value for size of VNF internal memory area
 296  * used for simulating noisy neighbour behaviour
 297  */
 298 uint64_t noisy_lkup_mem_sz;
 299
 300 /*
 301  * Configurable value of number of random writes done in
 302  * VNF simulation memory area.
 303  */
 304 uint64_t noisy_lkup_num_writes;
 305
 306 /*
 307  * Configurable value of number of random reads done in
 308  * VNF simulation memory area.
 309  */
 310 uint64_t noisy_lkup_num_reads;
 311
 312 /*
 313  * Configurable value of number of random reads/writes done in
 314  * VNF simulation memory area.
 315  */
 316 uint64_t noisy_lkup_num_reads_writes;
 317
 318 /*
 319  * Receive Side Scaling (RSS) configuration.
 320  */
 321 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 322
 323 /*
 324  * Port topology configuration
 325  */
 326 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 327
 328 /*
 329  * Avoids to flush all the RX streams before starts forwarding.
 330  */
 331 uint8_t no_flush_rx = 0; /* flush by default */
 332
 333 /*
 334  * Flow API isolated mode.
 335  */
 336 uint8_t flow_isolate_all;
 337
 338 /*
 339  * Avoids to check link status when starting/stopping a port.
 340  */
 341 uint8_t no_link_check = 0; /* check by default */
 342
 343 /*
 344  * Don't automatically start all ports in interactive mode.
 345  */
 346 uint8_t no_device_start = 0;
 347
 348 /*
 349  * Enable link status change notification
 350  */
 351 uint8_t lsc_interrupt = 1; /* enabled by default */
 352
 353 /*
 354  * Enable device removal notification.
 355  */
 356 uint8_t rmv_interrupt = 1; /* enabled by default */
 357
 358 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 359
 360 /* After attach, port setup is called on event or by iterator */
 361 bool setup_on_probe_event = true;
 362
 363 /* Clear ptypes on port initialization. */
 364 uint8_t clear_ptypes = true;
 365
 366 /* Pretty printing of ethdev events */
 367 static const char * const eth_event_desc[] = {
 368         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 369         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 370         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 371         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 372         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 373         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 374         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 375         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 376         [RTE_ETH_EVENT_NEW] = "device probed",
 377         [RTE_ETH_EVENT_DESTROY] = "device released",
 378         [RTE_ETH_EVENT_MAX] = NULL,
 379 };
 380
 381 /*
 382  * Display or mask ether events
 383  * Default to all events except VF_MBOX
 384  */
 385 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 388                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 389                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 390                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 391                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV);
 392 /*
 393  * Decide if all memory are locked for performance.
 394  */
 395 int do_mlockall = 0;
 396
 397 /*
 398  * NIC bypass mode configuration options.
 399  */
 400
 401 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 402 /* The NIC bypass watchdog timeout. */
 403 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 404 #endif
 405
 406
 407 #ifdef RTE_LIBRTE_LATENCY_STATS
 408
 409 /*
 410  * Set when latency stats is enabled in the commandline
 411  */
 412 uint8_t latencystats_enabled;
 413
 414 /*
 415  * Lcore ID to serive latency statistics.
 416  */
 417 lcoreid_t latencystats_lcore_id = -1;
 418
 419 #endif
 420
 421 /*
 422  * Ethernet device configuration.
 423  */
 424 struct rte_eth_rxmode rx_mode = {
 425         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 426                 /**< Default maximum frame length. */
 427 };
 428
 429 struct rte_eth_txmode tx_mode = {
 430         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 431 };
 432
 433 struct rte_fdir_conf fdir_conf = {
 434         .mode = RTE_FDIR_MODE_NONE,
 435         .pballoc = RTE_FDIR_PBALLOC_64K,
 436         .status = RTE_FDIR_REPORT_STATUS,
 437         .mask = {
 438                 .vlan_tci_mask = 0xFFEF,
 439                 .ipv4_mask     = {
 440                         .src_ip = 0xFFFFFFFF,
 441                         .dst_ip = 0xFFFFFFFF,
 442                 },
 443                 .ipv6_mask     = {
 444                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 445                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 446                 },
 447                 .src_port_mask = 0xFFFF,
 448                 .dst_port_mask = 0xFFFF,
 449                 .mac_addr_byte_mask = 0xFF,
 450                 .tunnel_type_mask = 1,
 451                 .tunnel_id_mask = 0xFFFFFFFF,
 452         },
 453         .drop_queue = 127,
 454 };
 455
 456 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 457
 458 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 459 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 460
 461 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 462 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 463
 464 uint16_t nb_tx_queue_stats_mappings = 0;
 465 uint16_t nb_rx_queue_stats_mappings = 0;
 466
 467 /*
 468  * Display zero values by default for xstats
 469  */
 470 uint8_t xstats_hide_zero;
 471
 472 unsigned int num_sockets = 0;
 473 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 474
 475 #ifdef RTE_LIBRTE_BITRATE
 476 /* Bitrate statistics */
 477 struct rte_stats_bitrates *bitrate_data;
 478 lcoreid_t bitrate_lcore_id;
 479 uint8_t bitrate_enabled;
 480 #endif
 481
 482 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 483 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 484
 485 /* Forward function declarations */
 486 static void setup_attached_port(portid_t pi);
 487 static void map_port_queue_stats_mapping_registers(portid_t pi,
 488                                                    struct rte_port *port);
 489 static void check_all_ports_link_status(uint32_t port_mask);
 490 static int eth_event_callback(portid_t port_id,
 491                               enum rte_eth_event_type type,
 492                               void *param, void *ret_param);
 493 static void dev_event_callback(const char *device_name,
 494                                 enum rte_dev_event_type type,
 495                                 void *param);
 496
 497 /*
 498  * Check if all the ports are started.
 499  * If yes, return positive value. If not, return zero.
 500  */
 501 static int all_ports_started(void);
 502
 503 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 504 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 505
 506 /* Holds the registered mbuf dynamic flags names. */
 507 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 508
 509 /*
 510  * Helper function to check if socket is already discovered.
 511  * If yes, return positive value. If not, return zero.
 512  */
 513 int
 514 new_socket_id(unsigned int socket_id)
 515 {
 516         unsigned int i;
 517
 518         for (i = 0; i < num_sockets; i++) {
 519                 if (socket_ids[i] == socket_id)
 520                         return 0;
 521         }
 522         return 1;
 523 }
 524
 525 /*
 526  * Setup default configuration.
 527  */
 528 static void
 529 set_default_fwd_lcores_config(void)
 530 {
 531         unsigned int i;
 532         unsigned int nb_lc;
 533         unsigned int sock_num;
 534
 535         nb_lc = 0;
 536         for (i = 0; i < RTE_MAX_LCORE; i++) {
 537                 if (!rte_lcore_is_enabled(i))
 538                         continue;
 539                 sock_num = rte_lcore_to_socket_id(i);
 540                 if (new_socket_id(sock_num)) {
 541                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 542                                 rte_exit(EXIT_FAILURE,
 543                                          "Total sockets greater than %u\n",
 544                                          RTE_MAX_NUMA_NODES);
 545                         }
 546                         socket_ids[num_sockets++] = sock_num;
 547                 }
 548                 if (i == rte_get_master_lcore())
 549                         continue;
 550                 fwd_lcores_cpuids[nb_lc++] = i;
 551         }
 552         nb_lcores = (lcoreid_t) nb_lc;
 553         nb_cfg_lcores = nb_lcores;
 554         nb_fwd_lcores = 1;
 555 }
 556
 557 static void
 558 set_def_peer_eth_addrs(void)
 559 {
 560         portid_t i;
 561
 562         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 563                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 564                 peer_eth_addrs[i].addr_bytes[5] = i;
 565         }
 566 }
 567
 568 static void
 569 set_default_fwd_ports_config(void)
 570 {
 571         portid_t pt_id;
 572         int i = 0;
 573
 574         RTE_ETH_FOREACH_DEV(pt_id) {
 575                 fwd_ports_ids[i++] = pt_id;
 576
 577                 /* Update sockets info according to the attached device */
 578                 int socket_id = rte_eth_dev_socket_id(pt_id);
 579                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 580                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 581                                 rte_exit(EXIT_FAILURE,
 582                                          "Total sockets greater than %u\n",
 583                                          RTE_MAX_NUMA_NODES);
 584                         }
 585                         socket_ids[num_sockets++] = socket_id;
 586                 }
 587         }
 588
 589         nb_cfg_ports = nb_ports;
 590         nb_fwd_ports = nb_ports;
 591 }
 592
 593 void
 594 set_def_fwd_config(void)
 595 {
 596         set_default_fwd_lcores_config();
 597         set_def_peer_eth_addrs();
 598         set_default_fwd_ports_config();
 599 }
 600
 601 /* extremely pessimistic estimation of memory required to create a mempool */
 602 static int
 603 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 604 {
 605         unsigned int n_pages, mbuf_per_pg, leftover;
 606         uint64_t total_mem, mbuf_mem, obj_sz;
 607
 608         /* there is no good way to predict how much space the mempool will
 609          * occupy because it will allocate chunks on the fly, and some of those
 610          * will come from default DPDK memory while some will come from our
 611          * external memory, so just assume 128MB will be enough for everyone.
 612          */
 613         uint64_t hdr_mem = 128 << 20;
 614
 615         /* account for possible non-contiguousness */
 616         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 617         if (obj_sz > pgsz) {
 618                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 619                 return -1;
 620         }
 621
 622         mbuf_per_pg = pgsz / obj_sz;
 623         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 624         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 625
 626         mbuf_mem = n_pages * pgsz;
 627
 628         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 629
 630         if (total_mem > SIZE_MAX) {
 631                 TESTPMD_LOG(ERR, "Memory size too big\n");
 632                 return -1;
 633         }
 634         *out = (size_t)total_mem;
 635
 636         return 0;
 637 }
 638
 639 static int
 640 pagesz_flags(uint64_t page_sz)
 641 {
 642         /* as per mmap() manpage, all page sizes are log2 of page size
 643          * shifted by MAP_HUGE_SHIFT
 644          */
 645         int log2 = rte_log2_u64(page_sz);
 646
 647         return (log2 << HUGE_SHIFT);
 648 }
 649
 650 static void *
 651 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 652 {
 653         void *addr;
 654         int flags;
 655
 656         /* allocate anonymous hugepages */
 657         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 658         if (huge)
 659                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 660
 661         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 662         if (addr == MAP_FAILED)
 663                 return NULL;
 664
 665         return addr;
 666 }
 667
 668 struct extmem_param {
 669         void *addr;
 670         size_t len;
 671         size_t pgsz;
 672         rte_iova_t *iova_table;
 673         unsigned int iova_table_len;
 674 };
 675
 676 static int
 677 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 678                 bool huge)
 679 {
 680         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 681                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 682         unsigned int cur_page, n_pages, pgsz_idx;
 683         size_t mem_sz, cur_pgsz;
 684         rte_iova_t *iovas = NULL;
 685         void *addr;
 686         int ret;
 687
 688         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 689                 /* skip anything that is too big */
 690                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 691                         continue;
 692
 693                 cur_pgsz = pgsizes[pgsz_idx];
 694
 695                 /* if we were told not to allocate hugepages, override */
 696                 if (!huge)
 697                         cur_pgsz = sysconf(_SC_PAGESIZE);
 698
 699                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 700                 if (ret < 0) {
 701                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 702                         return -1;
 703                 }
 704
 705                 /* allocate our memory */
 706                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 707
 708                 /* if we couldn't allocate memory with a specified page size,
 709                  * that doesn't mean we can't do it with other page sizes, so
 710                  * try another one.
 711                  */
 712                 if (addr == NULL)
 713                         continue;
 714
 715                 /* store IOVA addresses for every page in this memory area */
 716                 n_pages = mem_sz / cur_pgsz;
 717
 718                 iovas = malloc(sizeof(*iovas) * n_pages);
 719
 720                 if (iovas == NULL) {
 721                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 722                         goto fail;
 723                 }
 724                 /* lock memory if it's not huge pages */
 725                 if (!huge)
 726                         mlock(addr, mem_sz);
 727
 728                 /* populate IOVA addresses */
 729                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 730                         rte_iova_t iova;
 731                         size_t offset;
 732                         void *cur;
 733
 734                         offset = cur_pgsz * cur_page;
 735                         cur = RTE_PTR_ADD(addr, offset);
 736
 737                         /* touch the page before getting its IOVA */
 738                         *(volatile char *)cur = 0;
 739
 740                         iova = rte_mem_virt2iova(cur);
 741
 742                         iovas[cur_page] = iova;
 743                 }
 744
 745                 break;
 746         }
 747         /* if we couldn't allocate anything */
 748         if (iovas == NULL)
 749                 return -1;
 750
 751         param->addr = addr;
 752         param->len = mem_sz;
 753         param->pgsz = cur_pgsz;
 754         param->iova_table = iovas;
 755         param->iova_table_len = n_pages;
 756
 757         return 0;
 758 fail:
 759         if (iovas)
 760                 free(iovas);
 761         if (addr)
 762                 munmap(addr, mem_sz);
 763
 764         return -1;
 765 }
 766
 767 static int
 768 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 769 {
 770         struct extmem_param param;
 771         int socket_id, ret;
 772
 773         memset(&param, 0, sizeof(param));
 774
 775         /* check if our heap exists */
 776         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 777         if (socket_id < 0) {
 778                 /* create our heap */
 779                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 780                 if (ret < 0) {
 781                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 782                         return -1;
 783                 }
 784         }
 785
 786         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 787         if (ret < 0) {
 788                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 789                 return -1;
 790         }
 791
 792         /* we now have a valid memory area, so add it to heap */
 793         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 794                         param.addr, param.len, param.iova_table,
 795                         param.iova_table_len, param.pgsz);
 796
 797         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 798
 799         /* not needed any more */
 800         free(param.iova_table);
 801
 802         if (ret < 0) {
 803                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 804                 munmap(param.addr, param.len);
 805                 return -1;
 806         }
 807
 808         /* success */
 809
 810         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 811                         param.len >> 20);
 812
 813         return 0;
 814 }
 815 static void
 816 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 817              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 818 {
 819         uint16_t pid = 0;
 820         int ret;
 821
 822         RTE_ETH_FOREACH_DEV(pid) {
 823                 struct rte_eth_dev *dev =
 824                         &rte_eth_devices[pid];
 825
 826                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 827                                         memhdr->len);
 828                 if (ret) {
 829                         TESTPMD_LOG(DEBUG,
 830                                     "unable to DMA unmap addr 0x%p "
 831                                     "for device %s\n",
 832                                     memhdr->addr, dev->data->name);
 833                 }
 834         }
 835         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 836         if (ret) {
 837                 TESTPMD_LOG(DEBUG,
 838                             "unable to un-register addr 0x%p\n", memhdr->addr);
 839         }
 840 }
 841
 842 static void
 843 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 844            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 845 {
 846         uint16_t pid = 0;
 847         size_t page_size = sysconf(_SC_PAGESIZE);
 848         int ret;
 849
 850         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 851                                   page_size);
 852         if (ret) {
 853                 TESTPMD_LOG(DEBUG,
 854                             "unable to register addr 0x%p\n", memhdr->addr);
 855                 return;
 856         }
 857         RTE_ETH_FOREACH_DEV(pid) {
 858                 struct rte_eth_dev *dev =
 859                         &rte_eth_devices[pid];
 860
 861                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 862                                       memhdr->len);
 863                 if (ret) {
 864                         TESTPMD_LOG(DEBUG,
 865                                     "unable to DMA map addr 0x%p "
 866                                     "for device %s\n",
 867                                     memhdr->addr, dev->data->name);
 868                 }
 869         }
 870 }
 871
 872 static unsigned int
 873 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 874             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 875 {
 876         struct rte_pktmbuf_extmem *xmem;
 877         unsigned int ext_num, zone_num, elt_num;
 878         uint16_t elt_size;
 879
 880         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 881         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 882         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 883
 884         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 885         if (xmem == NULL) {
 886                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 887                                  "external buffer descriptors\n");
 888                 *ext_mem = NULL;
 889                 return 0;
 890         }
 891         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 892                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 893                 const struct rte_memzone *mz;
 894                 char mz_name[RTE_MEMZONE_NAMESIZE];
 895                 int ret;
 896
 897                 ret = snprintf(mz_name, sizeof(mz_name),
 898                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 899                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 900                         errno = ENAMETOOLONG;
 901                         ext_num = 0;
 902                         break;
 903                 }
 904                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 905                                                  socket_id,
 906                                                  RTE_MEMZONE_IOVA_CONTIG |
 907                                                  RTE_MEMZONE_1GB |
 908                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 909                                                  EXTBUF_ZONE_SIZE);
 910                 if (mz == NULL) {
 911                         /*
 912                          * The caller exits on external buffer creation
 913                          * error, so there is no need to free memzones.
 914                          */
 915                         errno = ENOMEM;
 916                         ext_num = 0;
 917                         break;
 918                 }
 919                 xseg->buf_ptr = mz->addr;
 920                 xseg->buf_iova = mz->iova;
 921                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 922                 xseg->elt_size = elt_size;
 923         }
 924         if (ext_num == 0 && xmem != NULL) {
 925                 free(xmem);
 926                 xmem = NULL;
 927         }
 928         *ext_mem = xmem;
 929         return ext_num;
 930 }
 931
 932 /*
 933  * Configuration initialisation done once at init time.
 934  */
 935 static struct rte_mempool *
 936 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 937                  unsigned int socket_id)
 938 {
 939         char pool_name[RTE_MEMPOOL_NAMESIZE];
 940         struct rte_mempool *rte_mp = NULL;
 941         uint32_t mb_size;
 942
 943         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 944         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 945
 946         TESTPMD_LOG(INFO,
 947                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 948                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 949
 950         switch (mp_alloc_type) {
 951         case MP_ALLOC_NATIVE:
 952                 {
 953                         /* wrapper to rte_mempool_create() */
 954                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 955                                         rte_mbuf_best_mempool_ops());
 956                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 957                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 958                         break;
 959                 }
 960         case MP_ALLOC_ANON:
 961                 {
 962                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 963                                 mb_size, (unsigned int) mb_mempool_cache,
 964                                 sizeof(struct rte_pktmbuf_pool_private),
 965                                 socket_id, mempool_flags);
 966                         if (rte_mp == NULL)
 967                                 goto err;
 968
 969                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 970                                 rte_mempool_free(rte_mp);
 971                                 rte_mp = NULL;
 972                                 goto err;
 973                         }
 974                         rte_pktmbuf_pool_init(rte_mp, NULL);
 975                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 976                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 977                         break;
 978                 }
 979         case MP_ALLOC_XMEM:
 980         case MP_ALLOC_XMEM_HUGE:
 981                 {
 982                         int heap_socket;
 983                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 984
 985                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 986                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 987
 988                         heap_socket =
 989                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 990                         if (heap_socket < 0)
 991                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 992
 993                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 994                                         rte_mbuf_best_mempool_ops());
 995                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 996                                         mb_mempool_cache, 0, mbuf_seg_size,
 997                                         heap_socket);
 998                         break;
 999                 }
1000         case MP_ALLOC_XBUF:
1001                 {
1002                         struct rte_pktmbuf_extmem *ext_mem;
1003                         unsigned int ext_num;
1004
1005                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1006                                                socket_id, pool_name, &ext_mem);
1007                         if (ext_num == 0)
1008                                 rte_exit(EXIT_FAILURE,
1009                                          "Can't create pinned data buffers\n");
1010
1011                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1012                                         rte_mbuf_best_mempool_ops());
1013                         rte_mp = rte_pktmbuf_pool_create_extbuf
1014                                         (pool_name, nb_mbuf, mb_mempool_cache,
1015                                          0, mbuf_seg_size, socket_id,
1016                                          ext_mem, ext_num);
1017                         free(ext_mem);
1018                         break;
1019                 }
1020         default:
1021                 {
1022                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1023                 }
1024         }
1025
1026 err:
1027         if (rte_mp == NULL) {
1028                 rte_exit(EXIT_FAILURE,
1029                         "Creation of mbuf pool for socket %u failed: %s\n",
1030                         socket_id, rte_strerror(rte_errno));
1031         } else if (verbose_level > 0) {
1032                 rte_mempool_dump(stdout, rte_mp);
1033         }
1034         return rte_mp;
1035 }
1036
1037 /*
1038  * Check given socket id is valid or not with NUMA mode,
1039  * if valid, return 0, else return -1
1040  */
1041 static int
1042 check_socket_id(const unsigned int socket_id)
1043 {
1044         static int warning_once = 0;
1045
1046         if (new_socket_id(socket_id)) {
1047                 if (!warning_once && numa_support)
1048                         printf("Warning: NUMA should be configured manually by"
1049                                " using --port-numa-config and"
1050                                " --ring-numa-config parameters along with"
1051                                " --numa.\n");
1052                 warning_once = 1;
1053                 return -1;
1054         }
1055         return 0;
1056 }
1057
1058 /*
1059  * Get the allowed maximum number of RX queues.
1060  * *pid return the port id which has minimal value of
1061  * max_rx_queues in all ports.
1062  */
1063 queueid_t
1064 get_allowed_max_nb_rxq(portid_t *pid)
1065 {
1066         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1067         bool max_rxq_valid = false;
1068         portid_t pi;
1069         struct rte_eth_dev_info dev_info;
1070
1071         RTE_ETH_FOREACH_DEV(pi) {
1072                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1073                         continue;
1074
1075                 max_rxq_valid = true;
1076                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1077                         allowed_max_rxq = dev_info.max_rx_queues;
1078                         *pid = pi;
1079                 }
1080         }
1081         return max_rxq_valid ? allowed_max_rxq : 0;
1082 }
1083
1084 /*
1085  * Check input rxq is valid or not.
1086  * If input rxq is not greater than any of maximum number
1087  * of RX queues of all ports, it is valid.
1088  * if valid, return 0, else return -1
1089  */
1090 int
1091 check_nb_rxq(queueid_t rxq)
1092 {
1093         queueid_t allowed_max_rxq;
1094         portid_t pid = 0;
1095
1096         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1097         if (rxq > allowed_max_rxq) {
1098                 printf("Fail: input rxq (%u) can't be greater "
1099                        "than max_rx_queues (%u) of port %u\n",
1100                        rxq,
1101                        allowed_max_rxq,
1102                        pid);
1103                 return -1;
1104         }
1105         return 0;
1106 }
1107
1108 /*
1109  * Get the allowed maximum number of TX queues.
1110  * *pid return the port id which has minimal value of
1111  * max_tx_queues in all ports.
1112  */
1113 queueid_t
1114 get_allowed_max_nb_txq(portid_t *pid)
1115 {
1116         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1117         bool max_txq_valid = false;
1118         portid_t pi;
1119         struct rte_eth_dev_info dev_info;
1120
1121         RTE_ETH_FOREACH_DEV(pi) {
1122                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1123                         continue;
1124
1125                 max_txq_valid = true;
1126                 if (dev_info.max_tx_queues < allowed_max_txq) {
1127                         allowed_max_txq = dev_info.max_tx_queues;
1128                         *pid = pi;
1129                 }
1130         }
1131         return max_txq_valid ? allowed_max_txq : 0;
1132 }
1133
1134 /*
1135  * Check input txq is valid or not.
1136  * If input txq is not greater than any of maximum number
1137  * of TX queues of all ports, it is valid.
1138  * if valid, return 0, else return -1
1139  */
1140 int
1141 check_nb_txq(queueid_t txq)
1142 {
1143         queueid_t allowed_max_txq;
1144         portid_t pid = 0;
1145
1146         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1147         if (txq > allowed_max_txq) {
1148                 printf("Fail: input txq (%u) can't be greater "
1149                        "than max_tx_queues (%u) of port %u\n",
1150                        txq,
1151                        allowed_max_txq,
1152                        pid);
1153                 return -1;
1154         }
1155         return 0;
1156 }
1157
1158 /*
1159  * Get the allowed maximum number of RXDs of every rx queue.
1160  * *pid return the port id which has minimal value of
1161  * max_rxd in all queues of all ports.
1162  */
1163 static uint16_t
1164 get_allowed_max_nb_rxd(portid_t *pid)
1165 {
1166         uint16_t allowed_max_rxd = UINT16_MAX;
1167         portid_t pi;
1168         struct rte_eth_dev_info dev_info;
1169
1170         RTE_ETH_FOREACH_DEV(pi) {
1171                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1172                         continue;
1173
1174                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1175                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1176                         *pid = pi;
1177                 }
1178         }
1179         return allowed_max_rxd;
1180 }
1181
1182 /*
1183  * Get the allowed minimal number of RXDs of every rx queue.
1184  * *pid return the port id which has minimal value of
1185  * min_rxd in all queues of all ports.
1186  */
1187 static uint16_t
1188 get_allowed_min_nb_rxd(portid_t *pid)
1189 {
1190         uint16_t allowed_min_rxd = 0;
1191         portid_t pi;
1192         struct rte_eth_dev_info dev_info;
1193
1194         RTE_ETH_FOREACH_DEV(pi) {
1195                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1196                         continue;
1197
1198                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1199                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1200                         *pid = pi;
1201                 }
1202         }
1203
1204         return allowed_min_rxd;
1205 }
1206
1207 /*
1208  * Check input rxd is valid or not.
1209  * If input rxd is not greater than any of maximum number
1210  * of RXDs of every Rx queues and is not less than any of
1211  * minimal number of RXDs of every Rx queues, it is valid.
1212  * if valid, return 0, else return -1
1213  */
1214 int
1215 check_nb_rxd(queueid_t rxd)
1216 {
1217         uint16_t allowed_max_rxd;
1218         uint16_t allowed_min_rxd;
1219         portid_t pid = 0;
1220
1221         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1222         if (rxd > allowed_max_rxd) {
1223                 printf("Fail: input rxd (%u) can't be greater "
1224                        "than max_rxds (%u) of port %u\n",
1225                        rxd,
1226                        allowed_max_rxd,
1227                        pid);
1228                 return -1;
1229         }
1230
1231         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1232         if (rxd < allowed_min_rxd) {
1233                 printf("Fail: input rxd (%u) can't be less "
1234                        "than min_rxds (%u) of port %u\n",
1235                        rxd,
1236                        allowed_min_rxd,
1237                        pid);
1238                 return -1;
1239         }
1240
1241         return 0;
1242 }
1243
1244 /*
1245  * Get the allowed maximum number of TXDs of every rx queues.
1246  * *pid return the port id which has minimal value of
1247  * max_txd in every tx queue.
1248  */
1249 static uint16_t
1250 get_allowed_max_nb_txd(portid_t *pid)
1251 {
1252         uint16_t allowed_max_txd = UINT16_MAX;
1253         portid_t pi;
1254         struct rte_eth_dev_info dev_info;
1255
1256         RTE_ETH_FOREACH_DEV(pi) {
1257                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1258                         continue;
1259
1260                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1261                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1262                         *pid = pi;
1263                 }
1264         }
1265         return allowed_max_txd;
1266 }
1267
1268 /*
1269  * Get the allowed maximum number of TXDs of every tx queues.
1270  * *pid return the port id which has minimal value of
1271  * min_txd in every tx queue.
1272  */
1273 static uint16_t
1274 get_allowed_min_nb_txd(portid_t *pid)
1275 {
1276         uint16_t allowed_min_txd = 0;
1277         portid_t pi;
1278         struct rte_eth_dev_info dev_info;
1279
1280         RTE_ETH_FOREACH_DEV(pi) {
1281                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1282                         continue;
1283
1284                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1285                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1286                         *pid = pi;
1287                 }
1288         }
1289
1290         return allowed_min_txd;
1291 }
1292
1293 /*
1294  * Check input txd is valid or not.
1295  * If input txd is not greater than any of maximum number
1296  * of TXDs of every Rx queues, it is valid.
1297  * if valid, return 0, else return -1
1298  */
1299 int
1300 check_nb_txd(queueid_t txd)
1301 {
1302         uint16_t allowed_max_txd;
1303         uint16_t allowed_min_txd;
1304         portid_t pid = 0;
1305
1306         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1307         if (txd > allowed_max_txd) {
1308                 printf("Fail: input txd (%u) can't be greater "
1309                        "than max_txds (%u) of port %u\n",
1310                        txd,
1311                        allowed_max_txd,
1312                        pid);
1313                 return -1;
1314         }
1315
1316         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1317         if (txd < allowed_min_txd) {
1318                 printf("Fail: input txd (%u) can't be less "
1319                        "than min_txds (%u) of port %u\n",
1320                        txd,
1321                        allowed_min_txd,
1322                        pid);
1323                 return -1;
1324         }
1325         return 0;
1326 }
1327
1328
1329 /*
1330  * Get the allowed maximum number of hairpin queues.
1331  * *pid return the port id which has minimal value of
1332  * max_hairpin_queues in all ports.
1333  */
1334 queueid_t
1335 get_allowed_max_nb_hairpinq(portid_t *pid)
1336 {
1337         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1338         portid_t pi;
1339         struct rte_eth_hairpin_cap cap;
1340
1341         RTE_ETH_FOREACH_DEV(pi) {
1342                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1343                         *pid = pi;
1344                         return 0;
1345                 }
1346                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1347                         allowed_max_hairpinq = cap.max_nb_queues;
1348                         *pid = pi;
1349                 }
1350         }
1351         return allowed_max_hairpinq;
1352 }
1353
1354 /*
1355  * Check input hairpin is valid or not.
1356  * If input hairpin is not greater than any of maximum number
1357  * of hairpin queues of all ports, it is valid.
1358  * if valid, return 0, else return -1
1359  */
1360 int
1361 check_nb_hairpinq(queueid_t hairpinq)
1362 {
1363         queueid_t allowed_max_hairpinq;
1364         portid_t pid = 0;
1365
1366         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1367         if (hairpinq > allowed_max_hairpinq) {
1368                 printf("Fail: input hairpin (%u) can't be greater "
1369                        "than max_hairpin_queues (%u) of port %u\n",
1370                        hairpinq, allowed_max_hairpinq, pid);
1371                 return -1;
1372         }
1373         return 0;
1374 }
1375
1376 static void
1377 init_config(void)
1378 {
1379         portid_t pid;
1380         struct rte_port *port;
1381         struct rte_mempool *mbp;
1382         unsigned int nb_mbuf_per_pool;
1383         lcoreid_t  lc_id;
1384         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1385         struct rte_gro_param gro_param;
1386         uint32_t gso_types;
1387         uint16_t data_size;
1388         bool warning = 0;
1389         int k;
1390         int ret;
1391
1392         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1393
1394         /* Configuration of logical cores. */
1395         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1396                                 sizeof(struct fwd_lcore *) * nb_lcores,
1397                                 RTE_CACHE_LINE_SIZE);
1398         if (fwd_lcores == NULL) {
1399                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1400                                                         "failed\n", nb_lcores);
1401         }
1402         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1403                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1404                                                sizeof(struct fwd_lcore),
1405                                                RTE_CACHE_LINE_SIZE);
1406                 if (fwd_lcores[lc_id] == NULL) {
1407                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1408                                                                 "failed\n");
1409                 }
1410                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1411         }
1412
1413         RTE_ETH_FOREACH_DEV(pid) {
1414                 port = &ports[pid];
1415                 /* Apply default TxRx configuration for all ports */
1416                 port->dev_conf.txmode = tx_mode;
1417                 port->dev_conf.rxmode = rx_mode;
1418
1419                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1420                 if (ret != 0)
1421                         rte_exit(EXIT_FAILURE,
1422                                  "rte_eth_dev_info_get() failed\n");
1423
1424                 if (!(port->dev_info.tx_offload_capa &
1425                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1426                         port->dev_conf.txmode.offloads &=
1427                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1428                 if (numa_support) {
1429                         if (port_numa[pid] != NUMA_NO_CONFIG)
1430                                 port_per_socket[port_numa[pid]]++;
1431                         else {
1432                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1433
1434                                 /*
1435                                  * if socket_id is invalid,
1436                                  * set to the first available socket.
1437                                  */
1438                                 if (check_socket_id(socket_id) < 0)
1439                                         socket_id = socket_ids[0];
1440                                 port_per_socket[socket_id]++;
1441                         }
1442                 }
1443
1444                 /* Apply Rx offloads configuration */
1445                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1446                         port->rx_conf[k].offloads =
1447                                 port->dev_conf.rxmode.offloads;
1448                 /* Apply Tx offloads configuration */
1449                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1450                         port->tx_conf[k].offloads =
1451                                 port->dev_conf.txmode.offloads;
1452
1453                 /* set flag to initialize port/queue */
1454                 port->need_reconfig = 1;
1455                 port->need_reconfig_queues = 1;
1456                 port->tx_metadata = 0;
1457
1458                 /* Check for maximum number of segments per MTU. Accordingly
1459                  * update the mbuf data size.
1460                  */
1461                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1462                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1463                         data_size = rx_mode.max_rx_pkt_len /
1464                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1465
1466                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1467                                                         mbuf_data_size) {
1468                                 mbuf_data_size = data_size +
1469                                                  RTE_PKTMBUF_HEADROOM;
1470                                 warning = 1;
1471                         }
1472                 }
1473         }
1474
1475         if (warning)
1476                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1477                             mbuf_data_size);
1478
1479         /*
1480          * Create pools of mbuf.
1481          * If NUMA support is disabled, create a single pool of mbuf in
1482          * socket 0 memory by default.
1483          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1484          *
1485          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1486          * nb_txd can be configured at run time.
1487          */
1488         if (param_total_num_mbufs)
1489                 nb_mbuf_per_pool = param_total_num_mbufs;
1490         else {
1491                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1492                         (nb_lcores * mb_mempool_cache) +
1493                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1494                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1495         }
1496
1497         if (numa_support) {
1498                 uint8_t i;
1499
1500                 for (i = 0; i < num_sockets; i++)
1501                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1502                                                        nb_mbuf_per_pool,
1503                                                        socket_ids[i]);
1504         } else {
1505                 if (socket_num == UMA_NO_CONFIG)
1506                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1507                                                        nb_mbuf_per_pool, 0);
1508                 else
1509                         mempools[socket_num] = mbuf_pool_create
1510                                                         (mbuf_data_size,
1511                                                          nb_mbuf_per_pool,
1512                                                          socket_num);
1513         }
1514
1515         init_port_config();
1516
1517         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1518                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1519         /*
1520          * Records which Mbuf pool to use by each logical core, if needed.
1521          */
1522         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1523                 mbp = mbuf_pool_find(
1524                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1525
1526                 if (mbp == NULL)
1527                         mbp = mbuf_pool_find(0);
1528                 fwd_lcores[lc_id]->mbp = mbp;
1529                 /* initialize GSO context */
1530                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1531                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1532                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1533                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1534                         RTE_ETHER_CRC_LEN;
1535                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1536         }
1537
1538         /* Configuration of packet forwarding streams. */
1539         if (init_fwd_streams() < 0)
1540                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1541
1542         fwd_config_setup();
1543
1544         /* create a gro context for each lcore */
1545         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1546         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1547         gro_param.max_item_per_flow = MAX_PKT_BURST;
1548         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1549                 gro_param.socket_id = rte_lcore_to_socket_id(
1550                                 fwd_lcores_cpuids[lc_id]);
1551                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1552                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1553                         rte_exit(EXIT_FAILURE,
1554                                         "rte_gro_ctx_create() failed\n");
1555                 }
1556         }
1557
1558 #if defined RTE_LIBRTE_PMD_SOFTNIC
1559         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1560                 RTE_ETH_FOREACH_DEV(pid) {
1561                         port = &ports[pid];
1562                         const char *driver = port->dev_info.driver_name;
1563
1564                         if (strcmp(driver, "net_softnic") == 0)
1565                                 port->softport.fwd_lcore_arg = fwd_lcores;
1566                 }
1567         }
1568 #endif
1569
1570 }
1571
1572
1573 void
1574 reconfig(portid_t new_port_id, unsigned socket_id)
1575 {
1576         struct rte_port *port;
1577         int ret;
1578
1579         /* Reconfiguration of Ethernet ports. */
1580         port = &ports[new_port_id];
1581
1582         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1583         if (ret != 0)
1584                 return;
1585
1586         /* set flag to initialize port/queue */
1587         port->need_reconfig = 1;
1588         port->need_reconfig_queues = 1;
1589         port->socket_id = socket_id;
1590
1591         init_port_config();
1592 }
1593
1594
1595 int
1596 init_fwd_streams(void)
1597 {
1598         portid_t pid;
1599         struct rte_port *port;
1600         streamid_t sm_id, nb_fwd_streams_new;
1601         queueid_t q;
1602
1603         /* set socket id according to numa or not */
1604         RTE_ETH_FOREACH_DEV(pid) {
1605                 port = &ports[pid];
1606                 if (nb_rxq > port->dev_info.max_rx_queues) {
1607                         printf("Fail: nb_rxq(%d) is greater than "
1608                                 "max_rx_queues(%d)\n", nb_rxq,
1609                                 port->dev_info.max_rx_queues);
1610                         return -1;
1611                 }
1612                 if (nb_txq > port->dev_info.max_tx_queues) {
1613                         printf("Fail: nb_txq(%d) is greater than "
1614                                 "max_tx_queues(%d)\n", nb_txq,
1615                                 port->dev_info.max_tx_queues);
1616                         return -1;
1617                 }
1618                 if (numa_support) {
1619                         if (port_numa[pid] != NUMA_NO_CONFIG)
1620                                 port->socket_id = port_numa[pid];
1621                         else {
1622                                 port->socket_id = rte_eth_dev_socket_id(pid);
1623
1624                                 /*
1625                                  * if socket_id is invalid,
1626                                  * set to the first available socket.
1627                                  */
1628                                 if (check_socket_id(port->socket_id) < 0)
1629                                         port->socket_id = socket_ids[0];
1630                         }
1631                 }
1632                 else {
1633                         if (socket_num == UMA_NO_CONFIG)
1634                                 port->socket_id = 0;
1635                         else
1636                                 port->socket_id = socket_num;
1637                 }
1638         }
1639
1640         q = RTE_MAX(nb_rxq, nb_txq);
1641         if (q == 0) {
1642                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1643                 return -1;
1644         }
1645         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1646         if (nb_fwd_streams_new == nb_fwd_streams)
1647                 return 0;
1648         /* clear the old */
1649         if (fwd_streams != NULL) {
1650                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1651                         if (fwd_streams[sm_id] == NULL)
1652                                 continue;
1653                         rte_free(fwd_streams[sm_id]);
1654                         fwd_streams[sm_id] = NULL;
1655                 }
1656                 rte_free(fwd_streams);
1657                 fwd_streams = NULL;
1658         }
1659
1660         /* init new */
1661         nb_fwd_streams = nb_fwd_streams_new;
1662         if (nb_fwd_streams) {
1663                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1664                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1665                         RTE_CACHE_LINE_SIZE);
1666                 if (fwd_streams == NULL)
1667                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1668                                  " (struct fwd_stream *)) failed\n",
1669                                  nb_fwd_streams);
1670
1671                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1672                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1673                                 " struct fwd_stream", sizeof(struct fwd_stream),
1674                                 RTE_CACHE_LINE_SIZE);
1675                         if (fwd_streams[sm_id] == NULL)
1676                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1677                                          "(struct fwd_stream) failed\n");
1678                 }
1679         }
1680
1681         return 0;
1682 }
1683
1684 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1685 static void
1686 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1687 {
1688         unsigned int total_burst;
1689         unsigned int nb_burst;
1690         unsigned int burst_stats[3];
1691         uint16_t pktnb_stats[3];
1692         uint16_t nb_pkt;
1693         int burst_percent[3];
1694
1695         /*
1696          * First compute the total number of packet bursts and the
1697          * two highest numbers of bursts of the same number of packets.
1698          */
1699         total_burst = 0;
1700         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1701         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1702         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1703                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1704                 if (nb_burst == 0)
1705                         continue;
1706                 total_burst += nb_burst;
1707                 if (nb_burst > burst_stats[0]) {
1708                         burst_stats[1] = burst_stats[0];
1709                         pktnb_stats[1] = pktnb_stats[0];
1710                         burst_stats[0] = nb_burst;
1711                         pktnb_stats[0] = nb_pkt;
1712                 } else if (nb_burst > burst_stats[1]) {
1713                         burst_stats[1] = nb_burst;
1714                         pktnb_stats[1] = nb_pkt;
1715                 }
1716         }
1717         if (total_burst == 0)
1718                 return;
1719         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1720         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1721                burst_percent[0], (int) pktnb_stats[0]);
1722         if (burst_stats[0] == total_burst) {
1723                 printf("]\n");
1724                 return;
1725         }
1726         if (burst_stats[0] + burst_stats[1] == total_burst) {
1727                 printf(" + %d%% of %d pkts]\n",
1728                        100 - burst_percent[0], pktnb_stats[1]);
1729                 return;
1730         }
1731         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1732         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1733         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1734                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1735                 return;
1736         }
1737         printf(" + %d%% of %d pkts + %d%% of others]\n",
1738                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1739 }
1740 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1741
1742 static void
1743 fwd_stream_stats_display(streamid_t stream_id)
1744 {
1745         struct fwd_stream *fs;
1746         static const char *fwd_top_stats_border = "-------";
1747
1748         fs = fwd_streams[stream_id];
1749         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1750             (fs->fwd_dropped == 0))
1751                 return;
1752         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1753                "TX Port=%2d/Queue=%2d %s\n",
1754                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1755                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1756         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1757                " TX-dropped: %-14"PRIu64,
1758                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1759
1760         /* if checksum mode */
1761         if (cur_fwd_eng == &csum_fwd_engine) {
1762                 printf("  RX- bad IP checksum: %-14"PRIu64
1763                        "  Rx- bad L4 checksum: %-14"PRIu64
1764                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1765                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1766                         fs->rx_bad_outer_l4_csum);
1767         } else {
1768                 printf("\n");
1769         }
1770
1771 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1772         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1773         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1774 #endif
1775 }
1776
1777 void
1778 fwd_stats_display(void)
1779 {
1780         static const char *fwd_stats_border = "----------------------";
1781         static const char *acc_stats_border = "+++++++++++++++";
1782         struct {
1783                 struct fwd_stream *rx_stream;
1784                 struct fwd_stream *tx_stream;
1785                 uint64_t tx_dropped;
1786                 uint64_t rx_bad_ip_csum;
1787                 uint64_t rx_bad_l4_csum;
1788                 uint64_t rx_bad_outer_l4_csum;
1789         } ports_stats[RTE_MAX_ETHPORTS];
1790         uint64_t total_rx_dropped = 0;
1791         uint64_t total_tx_dropped = 0;
1792         uint64_t total_rx_nombuf = 0;
1793         struct rte_eth_stats stats;
1794 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1795         uint64_t fwd_cycles = 0;
1796 #endif
1797         uint64_t total_recv = 0;
1798         uint64_t total_xmit = 0;
1799         struct rte_port *port;
1800         streamid_t sm_id;
1801         portid_t pt_id;
1802         int i;
1803
1804         memset(ports_stats, 0, sizeof(ports_stats));
1805
1806         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1807                 struct fwd_stream *fs = fwd_streams[sm_id];
1808
1809                 if (cur_fwd_config.nb_fwd_streams >
1810                     cur_fwd_config.nb_fwd_ports) {
1811                         fwd_stream_stats_display(sm_id);
1812                 } else {
1813                         ports_stats[fs->tx_port].tx_stream = fs;
1814                         ports_stats[fs->rx_port].rx_stream = fs;
1815                 }
1816
1817                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1818
1819                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1820                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1821                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1822                                 fs->rx_bad_outer_l4_csum;
1823
1824 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1825                 fwd_cycles += fs->core_cycles;
1826 #endif
1827         }
1828         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1829                 uint8_t j;
1830
1831                 pt_id = fwd_ports_ids[i];
1832                 port = &ports[pt_id];
1833
1834                 rte_eth_stats_get(pt_id, &stats);
1835                 stats.ipackets -= port->stats.ipackets;
1836                 stats.opackets -= port->stats.opackets;
1837                 stats.ibytes -= port->stats.ibytes;
1838                 stats.obytes -= port->stats.obytes;
1839                 stats.imissed -= port->stats.imissed;
1840                 stats.oerrors -= port->stats.oerrors;
1841                 stats.rx_nombuf -= port->stats.rx_nombuf;
1842
1843                 total_recv += stats.ipackets;
1844                 total_xmit += stats.opackets;
1845                 total_rx_dropped += stats.imissed;
1846                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1847                 total_tx_dropped += stats.oerrors;
1848                 total_rx_nombuf  += stats.rx_nombuf;
1849
1850                 printf("\n  %s Forward statistics for port %-2d %s\n",
1851                        fwd_stats_border, pt_id, fwd_stats_border);
1852
1853                 if (!port->rx_queue_stats_mapping_enabled &&
1854                     !port->tx_queue_stats_mapping_enabled) {
1855                         printf("  RX-packets: %-14"PRIu64
1856                                " RX-dropped: %-14"PRIu64
1857                                "RX-total: %-"PRIu64"\n",
1858                                stats.ipackets, stats.imissed,
1859                                stats.ipackets + stats.imissed);
1860
1861                         if (cur_fwd_eng == &csum_fwd_engine)
1862                                 printf("  Bad-ipcsum: %-14"PRIu64
1863                                        " Bad-l4csum: %-14"PRIu64
1864                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1865                                        ports_stats[pt_id].rx_bad_ip_csum,
1866                                        ports_stats[pt_id].rx_bad_l4_csum,
1867                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1868                         if (stats.ierrors + stats.rx_nombuf > 0) {
1869                                 printf("  RX-error: %-"PRIu64"\n",
1870                                        stats.ierrors);
1871                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1872                                        stats.rx_nombuf);
1873                         }
1874
1875                         printf("  TX-packets: %-14"PRIu64
1876                                " TX-dropped: %-14"PRIu64
1877                                "TX-total: %-"PRIu64"\n",
1878                                stats.opackets, ports_stats[pt_id].tx_dropped,
1879                                stats.opackets + ports_stats[pt_id].tx_dropped);
1880                 } else {
1881                         printf("  RX-packets:             %14"PRIu64
1882                                "    RX-dropped:%14"PRIu64
1883                                "    RX-total:%14"PRIu64"\n",
1884                                stats.ipackets, stats.imissed,
1885                                stats.ipackets + stats.imissed);
1886
1887                         if (cur_fwd_eng == &csum_fwd_engine)
1888                                 printf("  Bad-ipcsum:%14"PRIu64
1889                                        "    Bad-l4csum:%14"PRIu64
1890                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1891                                        ports_stats[pt_id].rx_bad_ip_csum,
1892                                        ports_stats[pt_id].rx_bad_l4_csum,
1893                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1894                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1895                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1896                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1897                                        stats.rx_nombuf);
1898                         }
1899
1900                         printf("  TX-packets:             %14"PRIu64
1901                                "    TX-dropped:%14"PRIu64
1902                                "    TX-total:%14"PRIu64"\n",
1903                                stats.opackets, ports_stats[pt_id].tx_dropped,
1904                                stats.opackets + ports_stats[pt_id].tx_dropped);
1905                 }
1906
1907 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1908                 if (ports_stats[pt_id].rx_stream)
1909                         pkt_burst_stats_display("RX",
1910                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1911                 if (ports_stats[pt_id].tx_stream)
1912                         pkt_burst_stats_display("TX",
1913                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1914 #endif
1915
1916                 if (port->rx_queue_stats_mapping_enabled) {
1917                         printf("\n");
1918                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1919                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1920                                        "     RX-errors:%14"PRIu64
1921                                        "    RX-bytes:%14"PRIu64"\n",
1922                                        j, stats.q_ipackets[j],
1923                                        stats.q_errors[j], stats.q_ibytes[j]);
1924                         }
1925                         printf("\n");
1926                 }
1927                 if (port->tx_queue_stats_mapping_enabled) {
1928                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1929                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1930                                        "                                 TX-bytes:%14"
1931                                        PRIu64"\n",
1932                                        j, stats.q_opackets[j],
1933                                        stats.q_obytes[j]);
1934                         }
1935                 }
1936
1937                 printf("  %s--------------------------------%s\n",
1938                        fwd_stats_border, fwd_stats_border);
1939         }
1940
1941         printf("\n  %s Accumulated forward statistics for all ports"
1942                "%s\n",
1943                acc_stats_border, acc_stats_border);
1944         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1945                "%-"PRIu64"\n"
1946                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1947                "%-"PRIu64"\n",
1948                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1949                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1950         if (total_rx_nombuf > 0)
1951                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1952         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1953                "%s\n",
1954                acc_stats_border, acc_stats_border);
1955 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1956         if (total_recv > 0)
1957                 printf("\n  CPU cycles/packet=%u (total cycles="
1958                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1959                        (unsigned int)(fwd_cycles / total_recv),
1960                        fwd_cycles, total_recv);
1961 #endif
1962 }
1963
1964 void
1965 fwd_stats_reset(void)
1966 {
1967         streamid_t sm_id;
1968         portid_t pt_id;
1969         int i;
1970
1971         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1972                 pt_id = fwd_ports_ids[i];
1973                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1974         }
1975         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1976                 struct fwd_stream *fs = fwd_streams[sm_id];
1977
1978                 fs->rx_packets = 0;
1979                 fs->tx_packets = 0;
1980                 fs->fwd_dropped = 0;
1981                 fs->rx_bad_ip_csum = 0;
1982                 fs->rx_bad_l4_csum = 0;
1983                 fs->rx_bad_outer_l4_csum = 0;
1984
1985 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1986                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1987                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1988 #endif
1989 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1990                 fs->core_cycles = 0;
1991 #endif
1992         }
1993 }
1994
1995 static void
1996 flush_fwd_rx_queues(void)
1997 {
1998         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1999         portid_t  rxp;
2000         portid_t port_id;
2001         queueid_t rxq;
2002         uint16_t  nb_rx;
2003         uint16_t  i;
2004         uint8_t   j;
2005         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2006         uint64_t timer_period;
2007
2008         /* convert to number of cycles */
2009         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2010
2011         for (j = 0; j < 2; j++) {
2012                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2013                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2014                                 port_id = fwd_ports_ids[rxp];
2015                                 /**
2016                                 * testpmd can stuck in the below do while loop
2017                                 * if rte_eth_rx_burst() always returns nonzero
2018                                 * packets. So timer is added to exit this loop
2019                                 * after 1sec timer expiry.
2020                                 */
2021                                 prev_tsc = rte_rdtsc();
2022                                 do {
2023                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2024                                                 pkts_burst, MAX_PKT_BURST);
2025                                         for (i = 0; i < nb_rx; i++)
2026                                                 rte_pktmbuf_free(pkts_burst[i]);
2027
2028                                         cur_tsc = rte_rdtsc();
2029                                         diff_tsc = cur_tsc - prev_tsc;
2030                                         timer_tsc += diff_tsc;
2031                                 } while ((nb_rx > 0) &&
2032                                         (timer_tsc < timer_period));
2033                                 timer_tsc = 0;
2034                         }
2035                 }
2036                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2037         }
2038 }
2039
2040 static void
2041 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2042 {
2043         struct fwd_stream **fsm;
2044         streamid_t nb_fs;
2045         streamid_t sm_id;
2046 #ifdef RTE_LIBRTE_BITRATE
2047         uint64_t tics_per_1sec;
2048         uint64_t tics_datum;
2049         uint64_t tics_current;
2050         uint16_t i, cnt_ports;
2051
2052         cnt_ports = nb_ports;
2053         tics_datum = rte_rdtsc();
2054         tics_per_1sec = rte_get_timer_hz();
2055 #endif
2056         fsm = &fwd_streams[fc->stream_idx];
2057         nb_fs = fc->stream_nb;
2058         do {
2059                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2060                         (*pkt_fwd)(fsm[sm_id]);
2061 #ifdef RTE_LIBRTE_BITRATE
2062                 if (bitrate_enabled != 0 &&
2063                                 bitrate_lcore_id == rte_lcore_id()) {
2064                         tics_current = rte_rdtsc();
2065                         if (tics_current - tics_datum >= tics_per_1sec) {
2066                                 /* Periodic bitrate calculation */
2067                                 for (i = 0; i < cnt_ports; i++)
2068                                         rte_stats_bitrate_calc(bitrate_data,
2069                                                 ports_ids[i]);
2070                                 tics_datum = tics_current;
2071                         }
2072                 }
2073 #endif
2074 #ifdef RTE_LIBRTE_LATENCY_STATS
2075                 if (latencystats_enabled != 0 &&
2076                                 latencystats_lcore_id == rte_lcore_id())
2077                         rte_latencystats_update();
2078 #endif
2079
2080         } while (! fc->stopped);
2081 }
2082
2083 static int
2084 start_pkt_forward_on_core(void *fwd_arg)
2085 {
2086         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2087                              cur_fwd_config.fwd_eng->packet_fwd);
2088         return 0;
2089 }
2090
2091 /*
2092  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2093  * Used to start communication flows in network loopback test configurations.
2094  */
2095 static int
2096 run_one_txonly_burst_on_core(void *fwd_arg)
2097 {
2098         struct fwd_lcore *fwd_lc;
2099         struct fwd_lcore tmp_lcore;
2100
2101         fwd_lc = (struct fwd_lcore *) fwd_arg;
2102         tmp_lcore = *fwd_lc;
2103         tmp_lcore.stopped = 1;
2104         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2105         return 0;
2106 }
2107
2108 /*
2109  * Launch packet forwarding:
2110  *     - Setup per-port forwarding context.
2111  *     - launch logical cores with their forwarding configuration.
2112  */
2113 static void
2114 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2115 {
2116         port_fwd_begin_t port_fwd_begin;
2117         unsigned int i;
2118         unsigned int lc_id;
2119         int diag;
2120
2121         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2122         if (port_fwd_begin != NULL) {
2123                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2124                         (*port_fwd_begin)(fwd_ports_ids[i]);
2125         }
2126         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2127                 lc_id = fwd_lcores_cpuids[i];
2128                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2129                         fwd_lcores[i]->stopped = 0;
2130                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2131                                                      fwd_lcores[i], lc_id);
2132                         if (diag != 0)
2133                                 printf("launch lcore %u failed - diag=%d\n",
2134                                        lc_id, diag);
2135                 }
2136         }
2137 }
2138
2139 /*
2140  * Launch packet forwarding configuration.
2141  */
2142 void
2143 start_packet_forwarding(int with_tx_first)
2144 {
2145         port_fwd_begin_t port_fwd_begin;
2146         port_fwd_end_t  port_fwd_end;
2147         struct rte_port *port;
2148         unsigned int i;
2149         portid_t   pt_id;
2150
2151         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2152                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2153
2154         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2155                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2156
2157         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2158                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2159                 (!nb_rxq || !nb_txq))
2160                 rte_exit(EXIT_FAILURE,
2161                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2162                         cur_fwd_eng->fwd_mode_name);
2163
2164         if (all_ports_started() == 0) {
2165                 printf("Not all ports were started\n");
2166                 return;
2167         }
2168         if (test_done == 0) {
2169                 printf("Packet forwarding already started\n");
2170                 return;
2171         }
2172
2173
2174         if(dcb_test) {
2175                 for (i = 0; i < nb_fwd_ports; i++) {
2176                         pt_id = fwd_ports_ids[i];
2177                         port = &ports[pt_id];
2178                         if (!port->dcb_flag) {
2179                                 printf("In DCB mode, all forwarding ports must "
2180                                        "be configured in this mode.\n");
2181                                 return;
2182                         }
2183                 }
2184                 if (nb_fwd_lcores == 1) {
2185                         printf("In DCB mode,the nb forwarding cores "
2186                                "should be larger than 1.\n");
2187                         return;
2188                 }
2189         }
2190         test_done = 0;
2191
2192         fwd_config_setup();
2193
2194         if(!no_flush_rx)
2195                 flush_fwd_rx_queues();
2196
2197         pkt_fwd_config_display(&cur_fwd_config);
2198         rxtx_config_display();
2199
2200         fwd_stats_reset();
2201         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2202                 pt_id = fwd_ports_ids[i];
2203                 port = &ports[pt_id];
2204                 map_port_queue_stats_mapping_registers(pt_id, port);
2205         }
2206         if (with_tx_first) {
2207                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2208                 if (port_fwd_begin != NULL) {
2209                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2210                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2211                 }
2212                 while (with_tx_first--) {
2213                         launch_packet_forwarding(
2214                                         run_one_txonly_burst_on_core);
2215                         rte_eal_mp_wait_lcore();
2216                 }
2217                 port_fwd_end = tx_only_engine.port_fwd_end;
2218                 if (port_fwd_end != NULL) {
2219                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2220                                 (*port_fwd_end)(fwd_ports_ids[i]);
2221                 }
2222         }
2223         launch_packet_forwarding(start_pkt_forward_on_core);
2224 }
2225
2226 void
2227 stop_packet_forwarding(void)
2228 {
2229         port_fwd_end_t port_fwd_end;
2230         lcoreid_t lc_id;
2231         portid_t pt_id;
2232         int i;
2233
2234         if (test_done) {
2235                 printf("Packet forwarding not started\n");
2236                 return;
2237         }
2238         printf("Telling cores to stop...");
2239         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2240                 fwd_lcores[lc_id]->stopped = 1;
2241         printf("\nWaiting for lcores to finish...\n");
2242         rte_eal_mp_wait_lcore();
2243         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2244         if (port_fwd_end != NULL) {
2245                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2246                         pt_id = fwd_ports_ids[i];
2247                         (*port_fwd_end)(pt_id);
2248                 }
2249         }
2250
2251         fwd_stats_display();
2252
2253         printf("\nDone.\n");
2254         test_done = 1;
2255 }
2256
2257 void
2258 dev_set_link_up(portid_t pid)
2259 {
2260         if (rte_eth_dev_set_link_up(pid) < 0)
2261                 printf("\nSet link up fail.\n");
2262 }
2263
2264 void
2265 dev_set_link_down(portid_t pid)
2266 {
2267         if (rte_eth_dev_set_link_down(pid) < 0)
2268                 printf("\nSet link down fail.\n");
2269 }
2270
2271 static int
2272 all_ports_started(void)
2273 {
2274         portid_t pi;
2275         struct rte_port *port;
2276
2277         RTE_ETH_FOREACH_DEV(pi) {
2278                 port = &ports[pi];
2279                 /* Check if there is a port which is not started */
2280                 if ((port->port_status != RTE_PORT_STARTED) &&
2281                         (port->slave_flag == 0))
2282                         return 0;
2283         }
2284
2285         /* No port is not started */
2286         return 1;
2287 }
2288
2289 int
2290 port_is_stopped(portid_t port_id)
2291 {
2292         struct rte_port *port = &ports[port_id];
2293
2294         if ((port->port_status != RTE_PORT_STOPPED) &&
2295             (port->slave_flag == 0))
2296                 return 0;
2297         return 1;
2298 }
2299
2300 int
2301 all_ports_stopped(void)
2302 {
2303         portid_t pi;
2304
2305         RTE_ETH_FOREACH_DEV(pi) {
2306                 if (!port_is_stopped(pi))
2307                         return 0;
2308         }
2309
2310         return 1;
2311 }
2312
2313 int
2314 port_is_started(portid_t port_id)
2315 {
2316         if (port_id_is_invalid(port_id, ENABLED_WARN))
2317                 return 0;
2318
2319         if (ports[port_id].port_status != RTE_PORT_STARTED)
2320                 return 0;
2321
2322         return 1;
2323 }
2324
2325 /* Configure the Rx and Tx hairpin queues for the selected port. */
2326 static int
2327 setup_hairpin_queues(portid_t pi)
2328 {
2329         queueid_t qi;
2330         struct rte_eth_hairpin_conf hairpin_conf = {
2331                 .peer_count = 1,
2332         };
2333         int i;
2334         int diag;
2335         struct rte_port *port = &ports[pi];
2336
2337         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2338                 hairpin_conf.peers[0].port = pi;
2339                 hairpin_conf.peers[0].queue = i + nb_rxq;
2340                 diag = rte_eth_tx_hairpin_queue_setup
2341                         (pi, qi, nb_txd, &hairpin_conf);
2342                 i++;
2343                 if (diag == 0)
2344                         continue;
2345
2346                 /* Fail to setup rx queue, return */
2347                 if (rte_atomic16_cmpset(&(port->port_status),
2348                                         RTE_PORT_HANDLING,
2349                                         RTE_PORT_STOPPED) == 0)
2350                         printf("Port %d can not be set back "
2351                                         "to stopped\n", pi);
2352                 printf("Fail to configure port %d hairpin "
2353                                 "queues\n", pi);
2354                 /* try to reconfigure queues next time */
2355                 port->need_reconfig_queues = 1;
2356                 return -1;
2357         }
2358         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2359                 hairpin_conf.peers[0].port = pi;
2360                 hairpin_conf.peers[0].queue = i + nb_txq;
2361                 diag = rte_eth_rx_hairpin_queue_setup
2362                         (pi, qi, nb_rxd, &hairpin_conf);
2363                 i++;
2364                 if (diag == 0)
2365                         continue;
2366
2367                 /* Fail to setup rx queue, return */
2368                 if (rte_atomic16_cmpset(&(port->port_status),
2369                                         RTE_PORT_HANDLING,
2370                                         RTE_PORT_STOPPED) == 0)
2371                         printf("Port %d can not be set back "
2372                                         "to stopped\n", pi);
2373                 printf("Fail to configure port %d hairpin "
2374                                 "queues\n", pi);
2375                 /* try to reconfigure queues next time */
2376                 port->need_reconfig_queues = 1;
2377                 return -1;
2378         }
2379         return 0;
2380 }
2381
2382 int
2383 start_port(portid_t pid)
2384 {
2385         int diag, need_check_link_status = -1;
2386         portid_t pi;
2387         queueid_t qi;
2388         struct rte_port *port;
2389         struct rte_ether_addr mac_addr;
2390         struct rte_eth_hairpin_cap cap;
2391
2392         if (port_id_is_invalid(pid, ENABLED_WARN))
2393                 return 0;
2394
2395         if(dcb_config)
2396                 dcb_test = 1;
2397         RTE_ETH_FOREACH_DEV(pi) {
2398                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2399                         continue;
2400
2401                 need_check_link_status = 0;
2402                 port = &ports[pi];
2403                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2404                                                  RTE_PORT_HANDLING) == 0) {
2405                         printf("Port %d is now not stopped\n", pi);
2406                         continue;
2407                 }
2408
2409                 if (port->need_reconfig > 0) {
2410                         port->need_reconfig = 0;
2411
2412                         if (flow_isolate_all) {
2413                                 int ret = port_flow_isolate(pi, 1);
2414                                 if (ret) {
2415                                         printf("Failed to apply isolated"
2416                                                " mode on port %d\n", pi);
2417                                         return -1;
2418                                 }
2419                         }
2420                         configure_rxtx_dump_callbacks(0);
2421                         printf("Configuring Port %d (socket %u)\n", pi,
2422                                         port->socket_id);
2423                         if (nb_hairpinq > 0 &&
2424                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2425                                 printf("Port %d doesn't support hairpin "
2426                                        "queues\n", pi);
2427                                 return -1;
2428                         }
2429                         /* configure port */
2430                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2431                                                      nb_txq + nb_hairpinq,
2432                                                      &(port->dev_conf));
2433                         if (diag != 0) {
2434                                 if (rte_atomic16_cmpset(&(port->port_status),
2435                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2436                                         printf("Port %d can not be set back "
2437                                                         "to stopped\n", pi);
2438                                 printf("Fail to configure port %d\n", pi);
2439                                 /* try to reconfigure port next time */
2440                                 port->need_reconfig = 1;
2441                                 return -1;
2442                         }
2443                 }
2444                 if (port->need_reconfig_queues > 0) {
2445                         port->need_reconfig_queues = 0;
2446                         /* setup tx queues */
2447                         for (qi = 0; qi < nb_txq; qi++) {
2448                                 if ((numa_support) &&
2449                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2450                                         diag = rte_eth_tx_queue_setup(pi, qi,
2451                                                 port->nb_tx_desc[qi],
2452                                                 txring_numa[pi],
2453                                                 &(port->tx_conf[qi]));
2454                                 else
2455                                         diag = rte_eth_tx_queue_setup(pi, qi,
2456                                                 port->nb_tx_desc[qi],
2457                                                 port->socket_id,
2458                                                 &(port->tx_conf[qi]));
2459
2460                                 if (diag == 0)
2461                                         continue;
2462
2463                                 /* Fail to setup tx queue, return */
2464                                 if (rte_atomic16_cmpset(&(port->port_status),
2465                                                         RTE_PORT_HANDLING,
2466                                                         RTE_PORT_STOPPED) == 0)
2467                                         printf("Port %d can not be set back "
2468                                                         "to stopped\n", pi);
2469                                 printf("Fail to configure port %d tx queues\n",
2470                                        pi);
2471                                 /* try to reconfigure queues next time */
2472                                 port->need_reconfig_queues = 1;
2473                                 return -1;
2474                         }
2475                         for (qi = 0; qi < nb_rxq; qi++) {
2476                                 /* setup rx queues */
2477                                 if ((numa_support) &&
2478                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2479                                         struct rte_mempool * mp =
2480                                                 mbuf_pool_find(rxring_numa[pi]);
2481                                         if (mp == NULL) {
2482                                                 printf("Failed to setup RX queue:"
2483                                                         "No mempool allocation"
2484                                                         " on the socket %d\n",
2485                                                         rxring_numa[pi]);
2486                                                 return -1;
2487                                         }
2488
2489                                         diag = rte_eth_rx_queue_setup(pi, qi,
2490                                              port->nb_rx_desc[qi],
2491                                              rxring_numa[pi],
2492                                              &(port->rx_conf[qi]),
2493                                              mp);
2494                                 } else {
2495                                         struct rte_mempool *mp =
2496                                                 mbuf_pool_find(port->socket_id);
2497                                         if (mp == NULL) {
2498                                                 printf("Failed to setup RX queue:"
2499                                                         "No mempool allocation"
2500                                                         " on the socket %d\n",
2501                                                         port->socket_id);
2502                                                 return -1;
2503                                         }
2504                                         diag = rte_eth_rx_queue_setup(pi, qi,
2505                                              port->nb_rx_desc[qi],
2506                                              port->socket_id,
2507                                              &(port->rx_conf[qi]),
2508                                              mp);
2509                                 }
2510                                 if (diag == 0)
2511                                         continue;
2512
2513                                 /* Fail to setup rx queue, return */
2514                                 if (rte_atomic16_cmpset(&(port->port_status),
2515                                                         RTE_PORT_HANDLING,
2516                                                         RTE_PORT_STOPPED) == 0)
2517                                         printf("Port %d can not be set back "
2518                                                         "to stopped\n", pi);
2519                                 printf("Fail to configure port %d rx queues\n",
2520                                        pi);
2521                                 /* try to reconfigure queues next time */
2522                                 port->need_reconfig_queues = 1;
2523                                 return -1;
2524                         }
2525                         /* setup hairpin queues */
2526                         if (setup_hairpin_queues(pi) != 0)
2527                                 return -1;
2528                 }
2529                 configure_rxtx_dump_callbacks(verbose_level);
2530                 if (clear_ptypes) {
2531                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2532                                         NULL, 0);
2533                         if (diag < 0)
2534                                 printf(
2535                                 "Port %d: Failed to disable Ptype parsing\n",
2536                                 pi);
2537                 }
2538
2539                 /* start port */
2540                 if (rte_eth_dev_start(pi) < 0) {
2541                         printf("Fail to start port %d\n", pi);
2542
2543                         /* Fail to setup rx queue, return */
2544                         if (rte_atomic16_cmpset(&(port->port_status),
2545                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2546                                 printf("Port %d can not be set back to "
2547                                                         "stopped\n", pi);
2548                         continue;
2549                 }
2550
2551                 if (rte_atomic16_cmpset(&(port->port_status),
2552                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2553                         printf("Port %d can not be set into started\n", pi);
2554
2555                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2556                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2557                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2558                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2559                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2560
2561                 /* at least one port started, need checking link status */
2562                 need_check_link_status = 1;
2563         }
2564
2565         if (need_check_link_status == 1 && !no_link_check)
2566                 check_all_ports_link_status(RTE_PORT_ALL);
2567         else if (need_check_link_status == 0)
2568                 printf("Please stop the ports first\n");
2569
2570         printf("Done\n");
2571         return 0;
2572 }
2573
2574 void
2575 stop_port(portid_t pid)
2576 {
2577         portid_t pi;
2578         struct rte_port *port;
2579         int need_check_link_status = 0;
2580
2581         if (dcb_test) {
2582                 dcb_test = 0;
2583                 dcb_config = 0;
2584         }
2585
2586         if (port_id_is_invalid(pid, ENABLED_WARN))
2587                 return;
2588
2589         printf("Stopping ports...\n");
2590
2591         RTE_ETH_FOREACH_DEV(pi) {
2592                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2593                         continue;
2594
2595                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2596                         printf("Please remove port %d from forwarding configuration.\n", pi);
2597                         continue;
2598                 }
2599
2600                 if (port_is_bonding_slave(pi)) {
2601                         printf("Please remove port %d from bonded device.\n", pi);
2602                         continue;
2603                 }
2604
2605                 port = &ports[pi];
2606                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2607                                                 RTE_PORT_HANDLING) == 0)
2608                         continue;
2609
2610                 rte_eth_dev_stop(pi);
2611
2612                 if (rte_atomic16_cmpset(&(port->port_status),
2613                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2614                         printf("Port %d can not be set into stopped\n", pi);
2615                 need_check_link_status = 1;
2616         }
2617         if (need_check_link_status && !no_link_check)
2618                 check_all_ports_link_status(RTE_PORT_ALL);
2619
2620         printf("Done\n");
2621 }
2622
2623 static void
2624 remove_invalid_ports_in(portid_t *array, portid_t *total)
2625 {
2626         portid_t i;
2627         portid_t new_total = 0;
2628
2629         for (i = 0; i < *total; i++)
2630                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2631                         array[new_total] = array[i];
2632                         new_total++;
2633                 }
2634         *total = new_total;
2635 }
2636
2637 static void
2638 remove_invalid_ports(void)
2639 {
2640         remove_invalid_ports_in(ports_ids, &nb_ports);
2641         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2642         nb_cfg_ports = nb_fwd_ports;
2643 }
2644
2645 void
2646 close_port(portid_t pid)
2647 {
2648         portid_t pi;
2649         struct rte_port *port;
2650
2651         if (port_id_is_invalid(pid, ENABLED_WARN))
2652                 return;
2653
2654         printf("Closing ports...\n");
2655
2656         RTE_ETH_FOREACH_DEV(pi) {
2657                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2658                         continue;
2659
2660                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2661                         printf("Please remove port %d from forwarding configuration.\n", pi);
2662                         continue;
2663                 }
2664
2665                 if (port_is_bonding_slave(pi)) {
2666                         printf("Please remove port %d from bonded device.\n", pi);
2667                         continue;
2668                 }
2669
2670                 port = &ports[pi];
2671                 if (rte_atomic16_cmpset(&(port->port_status),
2672                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2673                         printf("Port %d is already closed\n", pi);
2674                         continue;
2675                 }
2676
2677                 if (rte_atomic16_cmpset(&(port->port_status),
2678                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2679                         printf("Port %d is now not stopped\n", pi);
2680                         continue;
2681                 }
2682
2683                 if (port->flow_list)
2684                         port_flow_flush(pi);
2685                 rte_eth_dev_close(pi);
2686
2687                 remove_invalid_ports();
2688
2689                 if (rte_atomic16_cmpset(&(port->port_status),
2690                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2691                         printf("Port %d cannot be set to closed\n", pi);
2692         }
2693
2694         printf("Done\n");
2695 }
2696
2697 void
2698 reset_port(portid_t pid)
2699 {
2700         int diag;
2701         portid_t pi;
2702         struct rte_port *port;
2703
2704         if (port_id_is_invalid(pid, ENABLED_WARN))
2705                 return;
2706
2707         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2708                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2709                 printf("Can not reset port(s), please stop port(s) first.\n");
2710                 return;
2711         }
2712
2713         printf("Resetting ports...\n");
2714
2715         RTE_ETH_FOREACH_DEV(pi) {
2716                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2717                         continue;
2718
2719                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2720                         printf("Please remove port %d from forwarding "
2721                                "configuration.\n", pi);
2722                         continue;
2723                 }
2724
2725                 if (port_is_bonding_slave(pi)) {
2726                         printf("Please remove port %d from bonded device.\n",
2727                                pi);
2728                         continue;
2729                 }
2730
2731                 diag = rte_eth_dev_reset(pi);
2732                 if (diag == 0) {
2733                         port = &ports[pi];
2734                         port->need_reconfig = 1;
2735                         port->need_reconfig_queues = 1;
2736                 } else {
2737                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2738                 }
2739         }
2740
2741         printf("Done\n");
2742 }
2743
2744 void
2745 attach_port(char *identifier)
2746 {
2747         portid_t pi;
2748         struct rte_dev_iterator iterator;
2749
2750         printf("Attaching a new port...\n");
2751
2752         if (identifier == NULL) {
2753                 printf("Invalid parameters are specified\n");
2754                 return;
2755         }
2756
2757         if (rte_dev_probe(identifier) < 0) {
2758                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2759                 return;
2760         }
2761
2762         /* first attach mode: event */
2763         if (setup_on_probe_event) {
2764                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2765                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2766                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2767                                         ports[pi].need_setup != 0)
2768                                 setup_attached_port(pi);
2769                 return;
2770         }
2771
2772         /* second attach mode: iterator */
2773         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2774                 /* setup ports matching the devargs used for probing */
2775                 if (port_is_forwarding(pi))
2776                         continue; /* port was already attached before */
2777                 setup_attached_port(pi);
2778         }
2779 }
2780
2781 static void
2782 setup_attached_port(portid_t pi)
2783 {
2784         unsigned int socket_id;
2785         int ret;
2786
2787         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2788         /* if socket_id is invalid, set to the first available socket. */
2789         if (check_socket_id(socket_id) < 0)
2790                 socket_id = socket_ids[0];
2791         reconfig(pi, socket_id);
2792         ret = rte_eth_promiscuous_enable(pi);
2793         if (ret != 0)
2794                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2795                         pi, rte_strerror(-ret));
2796
2797         ports_ids[nb_ports++] = pi;
2798         fwd_ports_ids[nb_fwd_ports++] = pi;
2799         nb_cfg_ports = nb_fwd_ports;
2800         ports[pi].need_setup = 0;
2801         ports[pi].port_status = RTE_PORT_STOPPED;
2802
2803         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2804         printf("Done\n");
2805 }
2806
2807 static void
2808 detach_device(struct rte_device *dev)
2809 {
2810         portid_t sibling;
2811
2812         if (dev == NULL) {
2813                 printf("Device already removed\n");
2814                 return;
2815         }
2816
2817         printf("Removing a device...\n");
2818
2819         if (rte_dev_remove(dev) < 0) {
2820                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2821                 return;
2822         }
2823         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2824                 /* reset mapping between old ports and removed device */
2825                 rte_eth_devices[sibling].device = NULL;
2826                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2827                         /* sibling ports are forced to be closed */
2828                         ports[sibling].port_status = RTE_PORT_CLOSED;
2829                         printf("Port %u is closed\n", sibling);
2830                 }
2831         }
2832
2833         remove_invalid_ports();
2834
2835         printf("Device is detached\n");
2836         printf("Now total ports is %d\n", nb_ports);
2837         printf("Done\n");
2838         return;
2839 }
2840
2841 void
2842 detach_port_device(portid_t port_id)
2843 {
2844         if (port_id_is_invalid(port_id, ENABLED_WARN))
2845                 return;
2846
2847         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2848                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2849                         printf("Port not stopped\n");
2850                         return;
2851                 }
2852                 printf("Port was not closed\n");
2853                 if (ports[port_id].flow_list)
2854                         port_flow_flush(port_id);
2855         }
2856
2857         detach_device(rte_eth_devices[port_id].device);
2858 }
2859
2860 void
2861 detach_devargs(char *identifier)
2862 {
2863         struct rte_dev_iterator iterator;
2864         struct rte_devargs da;
2865         portid_t port_id;
2866
2867         printf("Removing a device...\n");
2868
2869         memset(&da, 0, sizeof(da));
2870         if (rte_devargs_parsef(&da, "%s", identifier)) {
2871                 printf("cannot parse identifier\n");
2872                 if (da.args)
2873                         free(da.args);
2874                 return;
2875         }
2876
2877         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2878                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2879                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2880                                 printf("Port %u not stopped\n", port_id);
2881                                 rte_eth_iterator_cleanup(&iterator);
2882                                 return;
2883                         }
2884
2885                         /* sibling ports are forced to be closed */
2886                         if (ports[port_id].flow_list)
2887                                 port_flow_flush(port_id);
2888                         ports[port_id].port_status = RTE_PORT_CLOSED;
2889                         printf("Port %u is now closed\n", port_id);
2890                 }
2891         }
2892
2893         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2894                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2895                             da.name, da.bus->name);
2896                 return;
2897         }
2898
2899         remove_invalid_ports();
2900
2901         printf("Device %s is detached\n", identifier);
2902         printf("Now total ports is %d\n", nb_ports);
2903         printf("Done\n");
2904 }
2905
2906 void
2907 pmd_test_exit(void)
2908 {
2909         portid_t pt_id;
2910         int ret;
2911         int i;
2912
2913         if (test_done == 0)
2914                 stop_packet_forwarding();
2915
2916         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2917                 if (mempools[i]) {
2918                         if (mp_alloc_type == MP_ALLOC_ANON)
2919                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2920                                                      NULL);
2921                 }
2922         }
2923         if (ports != NULL) {
2924                 no_link_check = 1;
2925                 RTE_ETH_FOREACH_DEV(pt_id) {
2926                         printf("\nStopping port %d...\n", pt_id);
2927                         fflush(stdout);
2928                         stop_port(pt_id);
2929                 }
2930                 RTE_ETH_FOREACH_DEV(pt_id) {
2931                         printf("\nShutting down port %d...\n", pt_id);
2932                         fflush(stdout);
2933                         close_port(pt_id);
2934                 }
2935         }
2936
2937         if (hot_plug) {
2938                 ret = rte_dev_event_monitor_stop();
2939                 if (ret) {
2940                         RTE_LOG(ERR, EAL,
2941                                 "fail to stop device event monitor.");
2942                         return;
2943                 }
2944
2945                 ret = rte_dev_event_callback_unregister(NULL,
2946                         dev_event_callback, NULL);
2947                 if (ret < 0) {
2948                         RTE_LOG(ERR, EAL,
2949                                 "fail to unregister device event callback.\n");
2950                         return;
2951                 }
2952
2953                 ret = rte_dev_hotplug_handle_disable();
2954                 if (ret) {
2955                         RTE_LOG(ERR, EAL,
2956                                 "fail to disable hotplug handling.\n");
2957                         return;
2958                 }
2959         }
2960         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2961                 if (mempools[i])
2962                         rte_mempool_free(mempools[i]);
2963         }
2964
2965         printf("\nBye...\n");
2966 }
2967
2968 typedef void (*cmd_func_t)(void);
2969 struct pmd_test_command {
2970         const char *cmd_name;
2971         cmd_func_t cmd_func;
2972 };
2973
2974 /* Check the link status of all ports in up to 9s, and print them finally */
2975 static void
2976 check_all_ports_link_status(uint32_t port_mask)
2977 {
2978 #define CHECK_INTERVAL 100 /* 100ms */
2979 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2980         portid_t portid;
2981         uint8_t count, all_ports_up, print_flag = 0;
2982         struct rte_eth_link link;
2983         int ret;
2984
2985         printf("Checking link statuses...\n");
2986         fflush(stdout);
2987         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2988                 all_ports_up = 1;
2989                 RTE_ETH_FOREACH_DEV(portid) {
2990                         if ((port_mask & (1 << portid)) == 0)
2991                                 continue;
2992                         memset(&link, 0, sizeof(link));
2993                         ret = rte_eth_link_get_nowait(portid, &link);
2994                         if (ret < 0) {
2995                                 all_ports_up = 0;
2996                                 if (print_flag == 1)
2997                                         printf("Port %u link get failed: %s\n",
2998                                                 portid, rte_strerror(-ret));
2999                                 continue;
3000                         }
3001                         /* print link status if flag set */
3002                         if (print_flag == 1) {
3003                                 if (link.link_status)
3004                                         printf(
3005                                         "Port%d Link Up. speed %u Mbps- %s\n",
3006                                         portid, link.link_speed,
3007                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3008                                         ("full-duplex") : ("half-duplex\n"));
3009                                 else
3010                                         printf("Port %d Link Down\n", portid);
3011                                 continue;
3012                         }
3013                         /* clear all_ports_up flag if any link down */
3014                         if (link.link_status == ETH_LINK_DOWN) {
3015                                 all_ports_up = 0;
3016                                 break;
3017                         }
3018                 }
3019                 /* after finally printing all link status, get out */
3020                 if (print_flag == 1)
3021                         break;
3022
3023                 if (all_ports_up == 0) {
3024                         fflush(stdout);
3025                         rte_delay_ms(CHECK_INTERVAL);
3026                 }
3027
3028                 /* set the print_flag if all ports up or timeout */
3029                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3030                         print_flag = 1;
3031                 }
3032
3033                 if (lsc_interrupt)
3034                         break;
3035         }
3036 }
3037
3038 /*
3039  * This callback is for remove a port for a device. It has limitation because
3040  * it is not for multiple port removal for a device.
3041  * TODO: the device detach invoke will plan to be removed from user side to
3042  * eal. And convert all PMDs to free port resources on ether device closing.
3043  */
3044 static void
3045 rmv_port_callback(void *arg)
3046 {
3047         int need_to_start = 0;
3048         int org_no_link_check = no_link_check;
3049         portid_t port_id = (intptr_t)arg;
3050         struct rte_device *dev;
3051
3052         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3053
3054         if (!test_done && port_is_forwarding(port_id)) {
3055                 need_to_start = 1;
3056                 stop_packet_forwarding();
3057         }
3058         no_link_check = 1;
3059         stop_port(port_id);
3060         no_link_check = org_no_link_check;
3061
3062         /* Save rte_device pointer before closing ethdev port */
3063         dev = rte_eth_devices[port_id].device;
3064         close_port(port_id);
3065         detach_device(dev); /* might be already removed or have more ports */
3066
3067         if (need_to_start)
3068                 start_packet_forwarding(0);
3069 }
3070
3071 /* This function is used by the interrupt thread */
3072 static int
3073 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3074                   void *ret_param)
3075 {
3076         RTE_SET_USED(param);
3077         RTE_SET_USED(ret_param);
3078
3079         if (type >= RTE_ETH_EVENT_MAX) {
3080                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3081                         port_id, __func__, type);
3082                 fflush(stderr);
3083         } else if (event_print_mask & (UINT32_C(1) << type)) {
3084                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3085                         eth_event_desc[type]);
3086                 fflush(stdout);
3087         }
3088
3089         switch (type) {
3090         case RTE_ETH_EVENT_NEW:
3091                 ports[port_id].need_setup = 1;
3092                 ports[port_id].port_status = RTE_PORT_HANDLING;
3093                 break;
3094         case RTE_ETH_EVENT_INTR_RMV:
3095                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3096                         break;
3097                 if (rte_eal_alarm_set(100000,
3098                                 rmv_port_callback, (void *)(intptr_t)port_id))
3099                         fprintf(stderr, "Could not set up deferred device removal\n");
3100                 break;
3101         default:
3102                 break;
3103         }
3104         return 0;
3105 }
3106
3107 static int
3108 register_eth_event_callback(void)
3109 {
3110         int ret;
3111         enum rte_eth_event_type event;
3112
3113         for (event = RTE_ETH_EVENT_UNKNOWN;
3114                         event < RTE_ETH_EVENT_MAX; event++) {
3115                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3116                                 event,
3117                                 eth_event_callback,
3118                                 NULL);
3119                 if (ret != 0) {
3120                         TESTPMD_LOG(ERR, "Failed to register callback for "
3121                                         "%s event\n", eth_event_desc[event]);
3122                         return -1;
3123                 }
3124         }
3125
3126         return 0;
3127 }
3128
3129 /* This function is used by the interrupt thread */
3130 static void
3131 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3132                              __rte_unused void *arg)
3133 {
3134         uint16_t port_id;
3135         int ret;
3136
3137         if (type >= RTE_DEV_EVENT_MAX) {
3138                 fprintf(stderr, "%s called upon invalid event %d\n",
3139                         __func__, type);
3140                 fflush(stderr);
3141         }
3142
3143         switch (type) {
3144         case RTE_DEV_EVENT_REMOVE:
3145                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3146                         device_name);
3147                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3148                 if (ret) {
3149                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3150                                 device_name);
3151                         return;
3152                 }
3153                 /*
3154                  * Because the user's callback is invoked in eal interrupt
3155                  * callback, the interrupt callback need to be finished before
3156                  * it can be unregistered when detaching device. So finish
3157                  * callback soon and use a deferred removal to detach device
3158                  * is need. It is a workaround, once the device detaching be
3159                  * moved into the eal in the future, the deferred removal could
3160                  * be deleted.
3161                  */
3162                 if (rte_eal_alarm_set(100000,
3163                                 rmv_port_callback, (void *)(intptr_t)port_id))
3164                         RTE_LOG(ERR, EAL,
3165                                 "Could not set up deferred device removal\n");
3166                 break;
3167         case RTE_DEV_EVENT_ADD:
3168                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3169                         device_name);
3170                 /* TODO: After finish kernel driver binding,
3171                  * begin to attach port.
3172                  */
3173                 break;
3174         default:
3175                 break;
3176         }
3177 }
3178
3179 static int
3180 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3181 {
3182         uint16_t i;
3183         int diag;
3184         uint8_t mapping_found = 0;
3185
3186         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3187                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3188                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3189                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3190                                         tx_queue_stats_mappings[i].queue_id,
3191                                         tx_queue_stats_mappings[i].stats_counter_id);
3192                         if (diag != 0)
3193                                 return diag;
3194                         mapping_found = 1;
3195                 }
3196         }
3197         if (mapping_found)
3198                 port->tx_queue_stats_mapping_enabled = 1;
3199         return 0;
3200 }
3201
3202 static int
3203 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3204 {
3205         uint16_t i;
3206         int diag;
3207         uint8_t mapping_found = 0;
3208
3209         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3210                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3211                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3212                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3213                                         rx_queue_stats_mappings[i].queue_id,
3214                                         rx_queue_stats_mappings[i].stats_counter_id);
3215                         if (diag != 0)
3216                                 return diag;
3217                         mapping_found = 1;
3218                 }
3219         }
3220         if (mapping_found)
3221                 port->rx_queue_stats_mapping_enabled = 1;
3222         return 0;
3223 }
3224
3225 static void
3226 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3227 {
3228         int diag = 0;
3229
3230         diag = set_tx_queue_stats_mapping_registers(pi, port);
3231         if (diag != 0) {
3232                 if (diag == -ENOTSUP) {
3233                         port->tx_queue_stats_mapping_enabled = 0;
3234                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3235                 }
3236                 else
3237                         rte_exit(EXIT_FAILURE,
3238                                         "set_tx_queue_stats_mapping_registers "
3239                                         "failed for port id=%d diag=%d\n",
3240                                         pi, diag);
3241         }
3242
3243         diag = set_rx_queue_stats_mapping_registers(pi, port);
3244         if (diag != 0) {
3245                 if (diag == -ENOTSUP) {
3246                         port->rx_queue_stats_mapping_enabled = 0;
3247                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3248                 }
3249                 else
3250                         rte_exit(EXIT_FAILURE,
3251                                         "set_rx_queue_stats_mapping_registers "
3252                                         "failed for port id=%d diag=%d\n",
3253                                         pi, diag);
3254         }
3255 }
3256
3257 static void
3258 rxtx_port_config(struct rte_port *port)
3259 {
3260         uint16_t qid;
3261         uint64_t offloads;
3262
3263         for (qid = 0; qid < nb_rxq; qid++) {
3264                 offloads = port->rx_conf[qid].offloads;
3265                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3266                 if (offloads != 0)
3267                         port->rx_conf[qid].offloads = offloads;
3268
3269                 /* Check if any Rx parameters have been passed */
3270                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3271                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3272
3273                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3274                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3275
3276                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3277                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3278
3279                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3280                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3281
3282                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3283                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3284
3285                 port->nb_rx_desc[qid] = nb_rxd;
3286         }
3287
3288         for (qid = 0; qid < nb_txq; qid++) {
3289                 offloads = port->tx_conf[qid].offloads;
3290                 port->tx_conf[qid] = port->dev_info.default_txconf;
3291                 if (offloads != 0)
3292                         port->tx_conf[qid].offloads = offloads;
3293
3294                 /* Check if any Tx parameters have been passed */
3295                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3296                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3297
3298                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3299                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3300
3301                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3302                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3303
3304                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3305                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3306
3307                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3308                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3309
3310                 port->nb_tx_desc[qid] = nb_txd;
3311         }
3312 }
3313
3314 void
3315 init_port_config(void)
3316 {
3317         portid_t pid;
3318         struct rte_port *port;
3319         int ret;
3320
3321         RTE_ETH_FOREACH_DEV(pid) {
3322                 port = &ports[pid];
3323                 port->dev_conf.fdir_conf = fdir_conf;
3324
3325                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3326                 if (ret != 0)
3327                         return;
3328
3329                 if (nb_rxq > 1) {
3330                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3331                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3332                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3333                 } else {
3334                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3335                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3336                 }
3337
3338                 if (port->dcb_flag == 0) {
3339                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3340                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
3341                         else
3342                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3343                 }
3344
3345                 rxtx_port_config(port);
3346
3347                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3348                 if (ret != 0)
3349                         return;
3350
3351                 map_port_queue_stats_mapping_registers(pid, port);
3352 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3353                 rte_pmd_ixgbe_bypass_init(pid);
3354 #endif
3355
3356                 if (lsc_interrupt &&
3357                     (rte_eth_devices[pid].data->dev_flags &
3358                      RTE_ETH_DEV_INTR_LSC))
3359                         port->dev_conf.intr_conf.lsc = 1;
3360                 if (rmv_interrupt &&
3361                     (rte_eth_devices[pid].data->dev_flags &
3362                      RTE_ETH_DEV_INTR_RMV))
3363                         port->dev_conf.intr_conf.rmv = 1;
3364         }
3365 }
3366
3367 void set_port_slave_flag(portid_t slave_pid)
3368 {
3369         struct rte_port *port;
3370
3371         port = &ports[slave_pid];
3372         port->slave_flag = 1;
3373 }
3374
3375 void clear_port_slave_flag(portid_t slave_pid)
3376 {
3377         struct rte_port *port;
3378
3379         port = &ports[slave_pid];
3380         port->slave_flag = 0;
3381 }
3382
3383 uint8_t port_is_bonding_slave(portid_t slave_pid)
3384 {
3385         struct rte_port *port;
3386
3387         port = &ports[slave_pid];
3388         if ((rte_eth_devices[slave_pid].data->dev_flags &
3389             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3390                 return 1;
3391         return 0;
3392 }
3393
3394 const uint16_t vlan_tags[] = {
3395                 0,  1,  2,  3,  4,  5,  6,  7,
3396                 8,  9, 10, 11,  12, 13, 14, 15,
3397                 16, 17, 18, 19, 20, 21, 22, 23,
3398                 24, 25, 26, 27, 28, 29, 30, 31
3399 };
3400
3401 static  int
3402 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3403                  enum dcb_mode_enable dcb_mode,
3404                  enum rte_eth_nb_tcs num_tcs,
3405                  uint8_t pfc_en)
3406 {
3407         uint8_t i;
3408         int32_t rc;
3409         struct rte_eth_rss_conf rss_conf;
3410
3411         /*
3412          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3413          * given above, and the number of traffic classes available for use.
3414          */
3415         if (dcb_mode == DCB_VT_ENABLED) {
3416                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3417                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3418                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3419                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3420
3421                 /* VMDQ+DCB RX and TX configurations */
3422                 vmdq_rx_conf->enable_default_pool = 0;
3423                 vmdq_rx_conf->default_pool = 0;
3424                 vmdq_rx_conf->nb_queue_pools =
3425                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3426                 vmdq_tx_conf->nb_queue_pools =
3427                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3428
3429                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3430                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3431                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3432                         vmdq_rx_conf->pool_map[i].pools =
3433                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3434                 }
3435                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3436                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3437                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3438                 }
3439
3440                 /* set DCB mode of RX and TX of multiple queues */
3441                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB;
3442                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3443         } else {
3444                 struct rte_eth_dcb_rx_conf *rx_conf =
3445                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3446                 struct rte_eth_dcb_tx_conf *tx_conf =
3447                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3448
3449                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3450                 if (rc != 0)
3451                         return rc;
3452
3453                 rx_conf->nb_tcs = num_tcs;
3454                 tx_conf->nb_tcs = num_tcs;
3455
3456                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3457                         rx_conf->dcb_tc[i] = i % num_tcs;
3458                         tx_conf->dcb_tc[i] = i % num_tcs;
3459                 }
3460
3461                 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS;
3462                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3463                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3464         }
3465
3466         if (pfc_en)
3467                 eth_conf->dcb_capability_en =
3468                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3469         else
3470                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3471
3472         return 0;
3473 }
3474
3475 int
3476 init_port_dcb_config(portid_t pid,
3477                      enum dcb_mode_enable dcb_mode,
3478                      enum rte_eth_nb_tcs num_tcs,
3479                      uint8_t pfc_en)
3480 {
3481         struct rte_eth_conf port_conf;
3482         struct rte_port *rte_port;
3483         int retval;
3484         uint16_t i;
3485
3486         rte_port = &ports[pid];
3487
3488         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3489         /* Enter DCB configuration status */
3490         dcb_config = 1;
3491
3492         port_conf.rxmode = rte_port->dev_conf.rxmode;
3493         port_conf.txmode = rte_port->dev_conf.txmode;
3494
3495         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3496         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3497         if (retval < 0)
3498                 return retval;
3499         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3500
3501         /* re-configure the device . */
3502         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3503         if (retval < 0)
3504                 return retval;
3505
3506         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3507         if (retval != 0)
3508                 return retval;
3509
3510         /* If dev_info.vmdq_pool_base is greater than 0,
3511          * the queue id of vmdq pools is started after pf queues.
3512          */
3513         if (dcb_mode == DCB_VT_ENABLED &&
3514             rte_port->dev_info.vmdq_pool_base > 0) {
3515                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3516                         " for port %d.", pid);
3517                 return -1;
3518         }
3519
3520         /* Assume the ports in testpmd have the same dcb capability
3521          * and has the same number of rxq and txq in dcb mode
3522          */
3523         if (dcb_mode == DCB_VT_ENABLED) {
3524                 if (rte_port->dev_info.max_vfs > 0) {
3525                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3526                         nb_txq = rte_port->dev_info.nb_tx_queues;
3527                 } else {
3528                         nb_rxq = rte_port->dev_info.max_rx_queues;
3529                         nb_txq = rte_port->dev_info.max_tx_queues;
3530                 }
3531         } else {
3532                 /*if vt is disabled, use all pf queues */
3533                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3534                         nb_rxq = rte_port->dev_info.max_rx_queues;
3535                         nb_txq = rte_port->dev_info.max_tx_queues;
3536                 } else {
3537                         nb_rxq = (queueid_t)num_tcs;
3538                         nb_txq = (queueid_t)num_tcs;
3539
3540                 }
3541         }
3542         rx_free_thresh = 64;
3543
3544         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3545
3546         rxtx_port_config(rte_port);
3547         /* VLAN filter */
3548         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3549         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3550                 rx_vft_set(pid, vlan_tags[i], 1);
3551
3552         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3553         if (retval != 0)
3554                 return retval;
3555
3556         map_port_queue_stats_mapping_registers(pid, rte_port);
3557
3558         rte_port->dcb_flag = 1;
3559
3560         return 0;
3561 }
3562
3563 static void
3564 init_port(void)
3565 {
3566         /* Configuration of Ethernet ports. */
3567         ports = rte_zmalloc("testpmd: ports",
3568                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3569                             RTE_CACHE_LINE_SIZE);
3570         if (ports == NULL) {
3571                 rte_exit(EXIT_FAILURE,
3572                                 "rte_zmalloc(%d struct rte_port) failed\n",
3573                                 RTE_MAX_ETHPORTS);
3574         }
3575
3576         /* Initialize ports NUMA structures */
3577         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3578         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3579         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3580 }
3581
3582 static void
3583 force_quit(void)
3584 {
3585         pmd_test_exit();
3586         prompt_exit();
3587 }
3588
3589 static void
3590 print_stats(void)
3591 {
3592         uint8_t i;
3593         const char clr[] = { 27, '[', '2', 'J', '\0' };
3594         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3595
3596         /* Clear screen and move to top left */
3597         printf("%s%s", clr, top_left);
3598
3599         printf("\nPort statistics ====================================");
3600         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3601                 nic_stats_display(fwd_ports_ids[i]);
3602
3603         fflush(stdout);
3604 }
3605
3606 static void
3607 signal_handler(int signum)
3608 {
3609         if (signum == SIGINT || signum == SIGTERM) {
3610                 printf("\nSignal %d received, preparing to exit...\n",
3611                                 signum);
3612 #ifdef RTE_LIBRTE_PDUMP
3613                 /* uninitialize packet capture framework */
3614                 rte_pdump_uninit();
3615 #endif
3616 #ifdef RTE_LIBRTE_LATENCY_STATS
3617                 if (latencystats_enabled != 0)
3618                         rte_latencystats_uninit();
3619 #endif
3620                 force_quit();
3621                 /* Set flag to indicate the force termination. */
3622                 f_quit = 1;
3623                 /* exit with the expected status */
3624                 signal(signum, SIG_DFL);
3625                 kill(getpid(), signum);
3626         }
3627 }
3628
3629 int
3630 main(int argc, char** argv)
3631 {
3632         int diag;
3633         portid_t port_id;
3634         uint16_t count;
3635         int ret;
3636
3637         signal(SIGINT, signal_handler);
3638         signal(SIGTERM, signal_handler);
3639
3640         testpmd_logtype = rte_log_register("testpmd");
3641         if (testpmd_logtype < 0)
3642                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3643         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3644
3645         diag = rte_eal_init(argc, argv);
3646         if (diag < 0)
3647                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3648                          rte_strerror(rte_errno));
3649
3650         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3651                 rte_exit(EXIT_FAILURE,
3652                          "Secondary process type not supported.\n");
3653
3654         ret = register_eth_event_callback();
3655         if (ret != 0)
3656                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3657
3658 #ifdef RTE_LIBRTE_PDUMP
3659         /* initialize packet capture framework */
3660         rte_pdump_init();
3661 #endif
3662
3663         count = 0;
3664         RTE_ETH_FOREACH_DEV(port_id) {
3665                 ports_ids[count] = port_id;
3666                 count++;
3667         }
3668         nb_ports = (portid_t) count;
3669         if (nb_ports == 0)
3670                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3671
3672         /* allocate port structures, and init them */
3673         init_port();
3674
3675         set_def_fwd_config();
3676         if (nb_lcores == 0)
3677                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3678                          "Check the core mask argument\n");
3679
3680         /* Bitrate/latency stats disabled by default */
3681 #ifdef RTE_LIBRTE_BITRATE
3682         bitrate_enabled = 0;
3683 #endif
3684 #ifdef RTE_LIBRTE_LATENCY_STATS
3685         latencystats_enabled = 0;
3686 #endif
3687
3688         /* on FreeBSD, mlockall() is disabled by default */
3689 #ifdef RTE_EXEC_ENV_FREEBSD
3690         do_mlockall = 0;
3691 #else
3692         do_mlockall = 1;
3693 #endif
3694
3695         argc -= diag;
3696         argv += diag;
3697         if (argc > 1)
3698                 launch_args_parse(argc, argv);
3699
3700         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3701                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3702                         strerror(errno));
3703         }
3704
3705         if (tx_first && interactive)
3706                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3707                                 "interactive mode.\n");
3708
3709         if (tx_first && lsc_interrupt) {
3710                 printf("Warning: lsc_interrupt needs to be off when "
3711                                 " using tx_first. Disabling.\n");
3712                 lsc_interrupt = 0;
3713         }
3714
3715         if (!nb_rxq && !nb_txq)
3716                 printf("Warning: Either rx or tx queues should be non-zero\n");
3717
3718         if (nb_rxq > 1 && nb_rxq > nb_txq)
3719                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3720                        "but nb_txq=%d will prevent to fully test it.\n",
3721                        nb_rxq, nb_txq);
3722
3723         init_config();
3724
3725         if (hot_plug) {
3726                 ret = rte_dev_hotplug_handle_enable();
3727                 if (ret) {
3728                         RTE_LOG(ERR, EAL,
3729                                 "fail to enable hotplug handling.");
3730                         return -1;
3731                 }
3732
3733                 ret = rte_dev_event_monitor_start();
3734                 if (ret) {
3735                         RTE_LOG(ERR, EAL,
3736                                 "fail to start device event monitoring.");
3737                         return -1;
3738                 }
3739
3740                 ret = rte_dev_event_callback_register(NULL,
3741                         dev_event_callback, NULL);
3742                 if (ret) {
3743                         RTE_LOG(ERR, EAL,
3744                                 "fail  to register device event callback\n");
3745                         return -1;
3746                 }
3747         }
3748
3749         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3750                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3751
3752         /* set all ports to promiscuous mode by default */
3753         RTE_ETH_FOREACH_DEV(port_id) {
3754                 ret = rte_eth_promiscuous_enable(port_id);
3755                 if (ret != 0)
3756                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3757                                 port_id, rte_strerror(-ret));
3758         }
3759
3760         /* Init metrics library */
3761         rte_metrics_init(rte_socket_id());
3762
3763 #ifdef RTE_LIBRTE_LATENCY_STATS
3764         if (latencystats_enabled != 0) {
3765                 int ret = rte_latencystats_init(1, NULL);
3766                 if (ret)
3767                         printf("Warning: latencystats init()"
3768                                 " returned error %d\n", ret);
3769                 printf("Latencystats running on lcore %d\n",
3770                         latencystats_lcore_id);
3771         }
3772 #endif
3773
3774         /* Setup bitrate stats */
3775 #ifdef RTE_LIBRTE_BITRATE
3776         if (bitrate_enabled != 0) {
3777                 bitrate_data = rte_stats_bitrate_create();
3778                 if (bitrate_data == NULL)
3779                         rte_exit(EXIT_FAILURE,
3780                                 "Could not allocate bitrate data.\n");
3781                 rte_stats_bitrate_reg(bitrate_data);
3782         }
3783 #endif
3784
3785 #ifdef RTE_LIBRTE_CMDLINE
3786         if (strlen(cmdline_filename) != 0)
3787                 cmdline_read_from_file(cmdline_filename);
3788
3789         if (interactive == 1) {
3790                 if (auto_start) {
3791                         printf("Start automatic packet forwarding\n");
3792                         start_packet_forwarding(0);
3793                 }
3794                 prompt();
3795                 pmd_test_exit();
3796         } else
3797 #endif
3798         {
3799                 char c;
3800                 int rc;
3801
3802                 f_quit = 0;
3803
3804                 printf("No commandline core given, start packet forwarding\n");
3805                 start_packet_forwarding(tx_first);
3806                 if (stats_period != 0) {
3807                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3808                         uint64_t timer_period;
3809
3810                         /* Convert to number of cycles */
3811                         timer_period = stats_period * rte_get_timer_hz();
3812
3813                         while (f_quit == 0) {
3814                                 cur_time = rte_get_timer_cycles();
3815                                 diff_time += cur_time - prev_time;
3816
3817                                 if (diff_time >= timer_period) {
3818                                         print_stats();
3819                                         /* Reset the timer */
3820                                         diff_time = 0;
3821                                 }
3822                                 /* Sleep to avoid unnecessary checks */
3823                                 prev_time = cur_time;
3824                                 sleep(1);
3825                         }
3826                 }
3827
3828                 printf("Press enter to exit\n");
3829                 rc = read(0, &c, 1);
3830                 pmd_test_exit();
3831                 if (rc < 0)
3832                         return 1;
3833         }
3834
3835         ret = rte_eal_cleanup();
3836         if (ret != 0)
3837                 rte_exit(EXIT_FAILURE,
3838                          "EAL cleanup failed: %s\n", strerror(-ret));
3839
3840         return EXIT_SUCCESS;
3841 }