app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 239 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 240 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 241
 242 /*
 243  * Configurable number of RX/TX ring descriptors.
 244  * Defaults are supplied by drivers via ethdev.
 245  */
 246 #define RTE_TEST_RX_DESC_DEFAULT 0
 247 #define RTE_TEST_TX_DESC_DEFAULT 0
 248 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 249 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 250
 251 #define RTE_PMD_PARAM_UNSET -1
 252 /*
 253  * Configurable values of RX and TX ring threshold registers.
 254  */
 255
 256 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 258 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 259
 260 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 262 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 263
 264 /*
 265  * Configurable value of RX free threshold.
 266  */
 267 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 268
 269 /*
 270  * Configurable value of RX drop enable.
 271  */
 272 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 273
 274 /*
 275  * Configurable value of TX free threshold.
 276  */
 277 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 278
 279 /*
 280  * Configurable value of TX RS bit threshold.
 281  */
 282 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 283
 284 /*
 285  * Configurable value of buffered packets before sending.
 286  */
 287 uint16_t noisy_tx_sw_bufsz;
 288
 289 /*
 290  * Configurable value of packet buffer timeout.
 291  */
 292 uint16_t noisy_tx_sw_buf_flush_time;
 293
 294 /*
 295  * Configurable value for size of VNF internal memory area
 296  * used for simulating noisy neighbour behaviour
 297  */
 298 uint64_t noisy_lkup_mem_sz;
 299
 300 /*
 301  * Configurable value of number of random writes done in
 302  * VNF simulation memory area.
 303  */
 304 uint64_t noisy_lkup_num_writes;
 305
 306 /*
 307  * Configurable value of number of random reads done in
 308  * VNF simulation memory area.
 309  */
 310 uint64_t noisy_lkup_num_reads;
 311
 312 /*
 313  * Configurable value of number of random reads/writes done in
 314  * VNF simulation memory area.
 315  */
 316 uint64_t noisy_lkup_num_reads_writes;
 317
 318 /*
 319  * Receive Side Scaling (RSS) configuration.
 320  */
 321 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 322
 323 /*
 324  * Port topology configuration
 325  */
 326 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 327
 328 /*
 329  * Avoids to flush all the RX streams before starts forwarding.
 330  */
 331 uint8_t no_flush_rx = 0; /* flush by default */
 332
 333 /*
 334  * Flow API isolated mode.
 335  */
 336 uint8_t flow_isolate_all;
 337
 338 /*
 339  * Avoids to check link status when starting/stopping a port.
 340  */
 341 uint8_t no_link_check = 0; /* check by default */
 342
 343 /*
 344  * Don't automatically start all ports in interactive mode.
 345  */
 346 uint8_t no_device_start = 0;
 347
 348 /*
 349  * Enable link status change notification
 350  */
 351 uint8_t lsc_interrupt = 1; /* enabled by default */
 352
 353 /*
 354  * Enable device removal notification.
 355  */
 356 uint8_t rmv_interrupt = 1; /* enabled by default */
 357
 358 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 359
 360 /* After attach, port setup is called on event or by iterator */
 361 bool setup_on_probe_event = true;
 362
 363 /* Clear ptypes on port initialization. */
 364 uint8_t clear_ptypes = true;
 365
 366 /* Pretty printing of ethdev events */
 367 static const char * const eth_event_desc[] = {
 368         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 369         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 370         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 371         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 372         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 373         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 374         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 375         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 376         [RTE_ETH_EVENT_NEW] = "device probed",
 377         [RTE_ETH_EVENT_DESTROY] = "device released",
 378         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 379         [RTE_ETH_EVENT_MAX] = NULL,
 380 };
 381
 382 /*
 383  * Display or mask ether events
 384  * Default to all events except VF_MBOX
 385  */
 386 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 388                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 389                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 390                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 391                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 392                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 393                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 394 /*
 395  * Decide if all memory are locked for performance.
 396  */
 397 int do_mlockall = 0;
 398
 399 /*
 400  * NIC bypass mode configuration options.
 401  */
 402
 403 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 404 /* The NIC bypass watchdog timeout. */
 405 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 406 #endif
 407
 408
 409 #ifdef RTE_LIBRTE_LATENCY_STATS
 410
 411 /*
 412  * Set when latency stats is enabled in the commandline
 413  */
 414 uint8_t latencystats_enabled;
 415
 416 /*
 417  * Lcore ID to serive latency statistics.
 418  */
 419 lcoreid_t latencystats_lcore_id = -1;
 420
 421 #endif
 422
 423 /*
 424  * Ethernet device configuration.
 425  */
 426 struct rte_eth_rxmode rx_mode = {
 427         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 428                 /**< Default maximum frame length. */
 429 };
 430
 431 struct rte_eth_txmode tx_mode = {
 432         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 433 };
 434
 435 struct rte_fdir_conf fdir_conf = {
 436         .mode = RTE_FDIR_MODE_NONE,
 437         .pballoc = RTE_FDIR_PBALLOC_64K,
 438         .status = RTE_FDIR_REPORT_STATUS,
 439         .mask = {
 440                 .vlan_tci_mask = 0xFFEF,
 441                 .ipv4_mask     = {
 442                         .src_ip = 0xFFFFFFFF,
 443                         .dst_ip = 0xFFFFFFFF,
 444                 },
 445                 .ipv6_mask     = {
 446                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 447                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 448                 },
 449                 .src_port_mask = 0xFFFF,
 450                 .dst_port_mask = 0xFFFF,
 451                 .mac_addr_byte_mask = 0xFF,
 452                 .tunnel_type_mask = 1,
 453                 .tunnel_id_mask = 0xFFFFFFFF,
 454         },
 455         .drop_queue = 127,
 456 };
 457
 458 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 459
 460 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 461 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 462
 463 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 464 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 465
 466 uint16_t nb_tx_queue_stats_mappings = 0;
 467 uint16_t nb_rx_queue_stats_mappings = 0;
 468
 469 /*
 470  * Display zero values by default for xstats
 471  */
 472 uint8_t xstats_hide_zero;
 473
 474 unsigned int num_sockets = 0;
 475 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 476
 477 #ifdef RTE_LIBRTE_BITRATE
 478 /* Bitrate statistics */
 479 struct rte_stats_bitrates *bitrate_data;
 480 lcoreid_t bitrate_lcore_id;
 481 uint8_t bitrate_enabled;
 482 #endif
 483
 484 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 485 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 486
 487 /*
 488  * hexadecimal bitmask of RX mq mode can be enabled.
 489  */
 490 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 491
 492 /* Forward function declarations */
 493 static void setup_attached_port(portid_t pi);
 494 static void map_port_queue_stats_mapping_registers(portid_t pi,
 495                                                    struct rte_port *port);
 496 static void check_all_ports_link_status(uint32_t port_mask);
 497 static int eth_event_callback(portid_t port_id,
 498                               enum rte_eth_event_type type,
 499                               void *param, void *ret_param);
 500 static void dev_event_callback(const char *device_name,
 501                                 enum rte_dev_event_type type,
 502                                 void *param);
 503
 504 /*
 505  * Check if all the ports are started.
 506  * If yes, return positive value. If not, return zero.
 507  */
 508 static int all_ports_started(void);
 509
 510 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 511 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 512
 513 /* Holds the registered mbuf dynamic flags names. */
 514 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 515
 516 /*
 517  * Helper function to check if socket is already discovered.
 518  * If yes, return positive value. If not, return zero.
 519  */
 520 int
 521 new_socket_id(unsigned int socket_id)
 522 {
 523         unsigned int i;
 524
 525         for (i = 0; i < num_sockets; i++) {
 526                 if (socket_ids[i] == socket_id)
 527                         return 0;
 528         }
 529         return 1;
 530 }
 531
 532 /*
 533  * Setup default configuration.
 534  */
 535 static void
 536 set_default_fwd_lcores_config(void)
 537 {
 538         unsigned int i;
 539         unsigned int nb_lc;
 540         unsigned int sock_num;
 541
 542         nb_lc = 0;
 543         for (i = 0; i < RTE_MAX_LCORE; i++) {
 544                 if (!rte_lcore_is_enabled(i))
 545                         continue;
 546                 sock_num = rte_lcore_to_socket_id(i);
 547                 if (new_socket_id(sock_num)) {
 548                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 549                                 rte_exit(EXIT_FAILURE,
 550                                          "Total sockets greater than %u\n",
 551                                          RTE_MAX_NUMA_NODES);
 552                         }
 553                         socket_ids[num_sockets++] = sock_num;
 554                 }
 555                 if (i == rte_get_master_lcore())
 556                         continue;
 557                 fwd_lcores_cpuids[nb_lc++] = i;
 558         }
 559         nb_lcores = (lcoreid_t) nb_lc;
 560         nb_cfg_lcores = nb_lcores;
 561         nb_fwd_lcores = 1;
 562 }
 563
 564 static void
 565 set_def_peer_eth_addrs(void)
 566 {
 567         portid_t i;
 568
 569         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 570                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 571                 peer_eth_addrs[i].addr_bytes[5] = i;
 572         }
 573 }
 574
 575 static void
 576 set_default_fwd_ports_config(void)
 577 {
 578         portid_t pt_id;
 579         int i = 0;
 580
 581         RTE_ETH_FOREACH_DEV(pt_id) {
 582                 fwd_ports_ids[i++] = pt_id;
 583
 584                 /* Update sockets info according to the attached device */
 585                 int socket_id = rte_eth_dev_socket_id(pt_id);
 586                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 587                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 588                                 rte_exit(EXIT_FAILURE,
 589                                          "Total sockets greater than %u\n",
 590                                          RTE_MAX_NUMA_NODES);
 591                         }
 592                         socket_ids[num_sockets++] = socket_id;
 593                 }
 594         }
 595
 596         nb_cfg_ports = nb_ports;
 597         nb_fwd_ports = nb_ports;
 598 }
 599
 600 void
 601 set_def_fwd_config(void)
 602 {
 603         set_default_fwd_lcores_config();
 604         set_def_peer_eth_addrs();
 605         set_default_fwd_ports_config();
 606 }
 607
 608 /* extremely pessimistic estimation of memory required to create a mempool */
 609 static int
 610 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 611 {
 612         unsigned int n_pages, mbuf_per_pg, leftover;
 613         uint64_t total_mem, mbuf_mem, obj_sz;
 614
 615         /* there is no good way to predict how much space the mempool will
 616          * occupy because it will allocate chunks on the fly, and some of those
 617          * will come from default DPDK memory while some will come from our
 618          * external memory, so just assume 128MB will be enough for everyone.
 619          */
 620         uint64_t hdr_mem = 128 << 20;
 621
 622         /* account for possible non-contiguousness */
 623         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 624         if (obj_sz > pgsz) {
 625                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 626                 return -1;
 627         }
 628
 629         mbuf_per_pg = pgsz / obj_sz;
 630         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 631         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 632
 633         mbuf_mem = n_pages * pgsz;
 634
 635         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 636
 637         if (total_mem > SIZE_MAX) {
 638                 TESTPMD_LOG(ERR, "Memory size too big\n");
 639                 return -1;
 640         }
 641         *out = (size_t)total_mem;
 642
 643         return 0;
 644 }
 645
 646 static int
 647 pagesz_flags(uint64_t page_sz)
 648 {
 649         /* as per mmap() manpage, all page sizes are log2 of page size
 650          * shifted by MAP_HUGE_SHIFT
 651          */
 652         int log2 = rte_log2_u64(page_sz);
 653
 654         return (log2 << HUGE_SHIFT);
 655 }
 656
 657 static void *
 658 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 659 {
 660         void *addr;
 661         int flags;
 662
 663         /* allocate anonymous hugepages */
 664         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 665         if (huge)
 666                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 667
 668         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 669         if (addr == MAP_FAILED)
 670                 return NULL;
 671
 672         return addr;
 673 }
 674
 675 struct extmem_param {
 676         void *addr;
 677         size_t len;
 678         size_t pgsz;
 679         rte_iova_t *iova_table;
 680         unsigned int iova_table_len;
 681 };
 682
 683 static int
 684 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 685                 bool huge)
 686 {
 687         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 688                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 689         unsigned int cur_page, n_pages, pgsz_idx;
 690         size_t mem_sz, cur_pgsz;
 691         rte_iova_t *iovas = NULL;
 692         void *addr;
 693         int ret;
 694
 695         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 696                 /* skip anything that is too big */
 697                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 698                         continue;
 699
 700                 cur_pgsz = pgsizes[pgsz_idx];
 701
 702                 /* if we were told not to allocate hugepages, override */
 703                 if (!huge)
 704                         cur_pgsz = sysconf(_SC_PAGESIZE);
 705
 706                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 707                 if (ret < 0) {
 708                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 709                         return -1;
 710                 }
 711
 712                 /* allocate our memory */
 713                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 714
 715                 /* if we couldn't allocate memory with a specified page size,
 716                  * that doesn't mean we can't do it with other page sizes, so
 717                  * try another one.
 718                  */
 719                 if (addr == NULL)
 720                         continue;
 721
 722                 /* store IOVA addresses for every page in this memory area */
 723                 n_pages = mem_sz / cur_pgsz;
 724
 725                 iovas = malloc(sizeof(*iovas) * n_pages);
 726
 727                 if (iovas == NULL) {
 728                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 729                         goto fail;
 730                 }
 731                 /* lock memory if it's not huge pages */
 732                 if (!huge)
 733                         mlock(addr, mem_sz);
 734
 735                 /* populate IOVA addresses */
 736                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 737                         rte_iova_t iova;
 738                         size_t offset;
 739                         void *cur;
 740
 741                         offset = cur_pgsz * cur_page;
 742                         cur = RTE_PTR_ADD(addr, offset);
 743
 744                         /* touch the page before getting its IOVA */
 745                         *(volatile char *)cur = 0;
 746
 747                         iova = rte_mem_virt2iova(cur);
 748
 749                         iovas[cur_page] = iova;
 750                 }
 751
 752                 break;
 753         }
 754         /* if we couldn't allocate anything */
 755         if (iovas == NULL)
 756                 return -1;
 757
 758         param->addr = addr;
 759         param->len = mem_sz;
 760         param->pgsz = cur_pgsz;
 761         param->iova_table = iovas;
 762         param->iova_table_len = n_pages;
 763
 764         return 0;
 765 fail:
 766         if (iovas)
 767                 free(iovas);
 768         if (addr)
 769                 munmap(addr, mem_sz);
 770
 771         return -1;
 772 }
 773
 774 static int
 775 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 776 {
 777         struct extmem_param param;
 778         int socket_id, ret;
 779
 780         memset(&param, 0, sizeof(param));
 781
 782         /* check if our heap exists */
 783         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 784         if (socket_id < 0) {
 785                 /* create our heap */
 786                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 787                 if (ret < 0) {
 788                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 789                         return -1;
 790                 }
 791         }
 792
 793         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 794         if (ret < 0) {
 795                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 796                 return -1;
 797         }
 798
 799         /* we now have a valid memory area, so add it to heap */
 800         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 801                         param.addr, param.len, param.iova_table,
 802                         param.iova_table_len, param.pgsz);
 803
 804         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 805
 806         /* not needed any more */
 807         free(param.iova_table);
 808
 809         if (ret < 0) {
 810                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 811                 munmap(param.addr, param.len);
 812                 return -1;
 813         }
 814
 815         /* success */
 816
 817         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 818                         param.len >> 20);
 819
 820         return 0;
 821 }
 822 static void
 823 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 824              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 825 {
 826         uint16_t pid = 0;
 827         int ret;
 828
 829         RTE_ETH_FOREACH_DEV(pid) {
 830                 struct rte_eth_dev *dev =
 831                         &rte_eth_devices[pid];
 832
 833                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 834                                         memhdr->len);
 835                 if (ret) {
 836                         TESTPMD_LOG(DEBUG,
 837                                     "unable to DMA unmap addr 0x%p "
 838                                     "for device %s\n",
 839                                     memhdr->addr, dev->data->name);
 840                 }
 841         }
 842         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 843         if (ret) {
 844                 TESTPMD_LOG(DEBUG,
 845                             "unable to un-register addr 0x%p\n", memhdr->addr);
 846         }
 847 }
 848
 849 static void
 850 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 851            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 852 {
 853         uint16_t pid = 0;
 854         size_t page_size = sysconf(_SC_PAGESIZE);
 855         int ret;
 856
 857         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 858                                   page_size);
 859         if (ret) {
 860                 TESTPMD_LOG(DEBUG,
 861                             "unable to register addr 0x%p\n", memhdr->addr);
 862                 return;
 863         }
 864         RTE_ETH_FOREACH_DEV(pid) {
 865                 struct rte_eth_dev *dev =
 866                         &rte_eth_devices[pid];
 867
 868                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 869                                       memhdr->len);
 870                 if (ret) {
 871                         TESTPMD_LOG(DEBUG,
 872                                     "unable to DMA map addr 0x%p "
 873                                     "for device %s\n",
 874                                     memhdr->addr, dev->data->name);
 875                 }
 876         }
 877 }
 878
 879 static unsigned int
 880 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 881             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 882 {
 883         struct rte_pktmbuf_extmem *xmem;
 884         unsigned int ext_num, zone_num, elt_num;
 885         uint16_t elt_size;
 886
 887         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 888         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 889         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 890
 891         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 892         if (xmem == NULL) {
 893                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 894                                  "external buffer descriptors\n");
 895                 *ext_mem = NULL;
 896                 return 0;
 897         }
 898         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 899                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 900                 const struct rte_memzone *mz;
 901                 char mz_name[RTE_MEMZONE_NAMESIZE];
 902                 int ret;
 903
 904                 ret = snprintf(mz_name, sizeof(mz_name),
 905                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 906                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 907                         errno = ENAMETOOLONG;
 908                         ext_num = 0;
 909                         break;
 910                 }
 911                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 912                                                  socket_id,
 913                                                  RTE_MEMZONE_IOVA_CONTIG |
 914                                                  RTE_MEMZONE_1GB |
 915                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 916                                                  EXTBUF_ZONE_SIZE);
 917                 if (mz == NULL) {
 918                         /*
 919                          * The caller exits on external buffer creation
 920                          * error, so there is no need to free memzones.
 921                          */
 922                         errno = ENOMEM;
 923                         ext_num = 0;
 924                         break;
 925                 }
 926                 xseg->buf_ptr = mz->addr;
 927                 xseg->buf_iova = mz->iova;
 928                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 929                 xseg->elt_size = elt_size;
 930         }
 931         if (ext_num == 0 && xmem != NULL) {
 932                 free(xmem);
 933                 xmem = NULL;
 934         }
 935         *ext_mem = xmem;
 936         return ext_num;
 937 }
 938
 939 /*
 940  * Configuration initialisation done once at init time.
 941  */
 942 static struct rte_mempool *
 943 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 944                  unsigned int socket_id)
 945 {
 946         char pool_name[RTE_MEMPOOL_NAMESIZE];
 947         struct rte_mempool *rte_mp = NULL;
 948         uint32_t mb_size;
 949
 950         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 951         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 952
 953         TESTPMD_LOG(INFO,
 954                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 955                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 956
 957         switch (mp_alloc_type) {
 958         case MP_ALLOC_NATIVE:
 959                 {
 960                         /* wrapper to rte_mempool_create() */
 961                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 962                                         rte_mbuf_best_mempool_ops());
 963                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 964                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 965                         break;
 966                 }
 967         case MP_ALLOC_ANON:
 968                 {
 969                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 970                                 mb_size, (unsigned int) mb_mempool_cache,
 971                                 sizeof(struct rte_pktmbuf_pool_private),
 972                                 socket_id, mempool_flags);
 973                         if (rte_mp == NULL)
 974                                 goto err;
 975
 976                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 977                                 rte_mempool_free(rte_mp);
 978                                 rte_mp = NULL;
 979                                 goto err;
 980                         }
 981                         rte_pktmbuf_pool_init(rte_mp, NULL);
 982                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 983                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 984                         break;
 985                 }
 986         case MP_ALLOC_XMEM:
 987         case MP_ALLOC_XMEM_HUGE:
 988                 {
 989                         int heap_socket;
 990                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 991
 992                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 993                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 994
 995                         heap_socket =
 996                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 997                         if (heap_socket < 0)
 998                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 999
1000                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1001                                         rte_mbuf_best_mempool_ops());
1002                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1003                                         mb_mempool_cache, 0, mbuf_seg_size,
1004                                         heap_socket);
1005                         break;
1006                 }
1007         case MP_ALLOC_XBUF:
1008                 {
1009                         struct rte_pktmbuf_extmem *ext_mem;
1010                         unsigned int ext_num;
1011
1012                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1013                                                socket_id, pool_name, &ext_mem);
1014                         if (ext_num == 0)
1015                                 rte_exit(EXIT_FAILURE,
1016                                          "Can't create pinned data buffers\n");
1017
1018                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1019                                         rte_mbuf_best_mempool_ops());
1020                         rte_mp = rte_pktmbuf_pool_create_extbuf
1021                                         (pool_name, nb_mbuf, mb_mempool_cache,
1022                                          0, mbuf_seg_size, socket_id,
1023                                          ext_mem, ext_num);
1024                         free(ext_mem);
1025                         break;
1026                 }
1027         default:
1028                 {
1029                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1030                 }
1031         }
1032
1033 err:
1034         if (rte_mp == NULL) {
1035                 rte_exit(EXIT_FAILURE,
1036                         "Creation of mbuf pool for socket %u failed: %s\n",
1037                         socket_id, rte_strerror(rte_errno));
1038         } else if (verbose_level > 0) {
1039                 rte_mempool_dump(stdout, rte_mp);
1040         }
1041         return rte_mp;
1042 }
1043
1044 /*
1045  * Check given socket id is valid or not with NUMA mode,
1046  * if valid, return 0, else return -1
1047  */
1048 static int
1049 check_socket_id(const unsigned int socket_id)
1050 {
1051         static int warning_once = 0;
1052
1053         if (new_socket_id(socket_id)) {
1054                 if (!warning_once && numa_support)
1055                         printf("Warning: NUMA should be configured manually by"
1056                                " using --port-numa-config and"
1057                                " --ring-numa-config parameters along with"
1058                                " --numa.\n");
1059                 warning_once = 1;
1060                 return -1;
1061         }
1062         return 0;
1063 }
1064
1065 /*
1066  * Get the allowed maximum number of RX queues.
1067  * *pid return the port id which has minimal value of
1068  * max_rx_queues in all ports.
1069  */
1070 queueid_t
1071 get_allowed_max_nb_rxq(portid_t *pid)
1072 {
1073         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1074         bool max_rxq_valid = false;
1075         portid_t pi;
1076         struct rte_eth_dev_info dev_info;
1077
1078         RTE_ETH_FOREACH_DEV(pi) {
1079                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1080                         continue;
1081
1082                 max_rxq_valid = true;
1083                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1084                         allowed_max_rxq = dev_info.max_rx_queues;
1085                         *pid = pi;
1086                 }
1087         }
1088         return max_rxq_valid ? allowed_max_rxq : 0;
1089 }
1090
1091 /*
1092  * Check input rxq is valid or not.
1093  * If input rxq is not greater than any of maximum number
1094  * of RX queues of all ports, it is valid.
1095  * if valid, return 0, else return -1
1096  */
1097 int
1098 check_nb_rxq(queueid_t rxq)
1099 {
1100         queueid_t allowed_max_rxq;
1101         portid_t pid = 0;
1102
1103         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1104         if (rxq > allowed_max_rxq) {
1105                 printf("Fail: input rxq (%u) can't be greater "
1106                        "than max_rx_queues (%u) of port %u\n",
1107                        rxq,
1108                        allowed_max_rxq,
1109                        pid);
1110                 return -1;
1111         }
1112         return 0;
1113 }
1114
1115 /*
1116  * Get the allowed maximum number of TX queues.
1117  * *pid return the port id which has minimal value of
1118  * max_tx_queues in all ports.
1119  */
1120 queueid_t
1121 get_allowed_max_nb_txq(portid_t *pid)
1122 {
1123         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1124         bool max_txq_valid = false;
1125         portid_t pi;
1126         struct rte_eth_dev_info dev_info;
1127
1128         RTE_ETH_FOREACH_DEV(pi) {
1129                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1130                         continue;
1131
1132                 max_txq_valid = true;
1133                 if (dev_info.max_tx_queues < allowed_max_txq) {
1134                         allowed_max_txq = dev_info.max_tx_queues;
1135                         *pid = pi;
1136                 }
1137         }
1138         return max_txq_valid ? allowed_max_txq : 0;
1139 }
1140
1141 /*
1142  * Check input txq is valid or not.
1143  * If input txq is not greater than any of maximum number
1144  * of TX queues of all ports, it is valid.
1145  * if valid, return 0, else return -1
1146  */
1147 int
1148 check_nb_txq(queueid_t txq)
1149 {
1150         queueid_t allowed_max_txq;
1151         portid_t pid = 0;
1152
1153         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1154         if (txq > allowed_max_txq) {
1155                 printf("Fail: input txq (%u) can't be greater "
1156                        "than max_tx_queues (%u) of port %u\n",
1157                        txq,
1158                        allowed_max_txq,
1159                        pid);
1160                 return -1;
1161         }
1162         return 0;
1163 }
1164
1165 /*
1166  * Get the allowed maximum number of RXDs of every rx queue.
1167  * *pid return the port id which has minimal value of
1168  * max_rxd in all queues of all ports.
1169  */
1170 static uint16_t
1171 get_allowed_max_nb_rxd(portid_t *pid)
1172 {
1173         uint16_t allowed_max_rxd = UINT16_MAX;
1174         portid_t pi;
1175         struct rte_eth_dev_info dev_info;
1176
1177         RTE_ETH_FOREACH_DEV(pi) {
1178                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1179                         continue;
1180
1181                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1182                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1183                         *pid = pi;
1184                 }
1185         }
1186         return allowed_max_rxd;
1187 }
1188
1189 /*
1190  * Get the allowed minimal number of RXDs of every rx queue.
1191  * *pid return the port id which has minimal value of
1192  * min_rxd in all queues of all ports.
1193  */
1194 static uint16_t
1195 get_allowed_min_nb_rxd(portid_t *pid)
1196 {
1197         uint16_t allowed_min_rxd = 0;
1198         portid_t pi;
1199         struct rte_eth_dev_info dev_info;
1200
1201         RTE_ETH_FOREACH_DEV(pi) {
1202                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1203                         continue;
1204
1205                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1206                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1207                         *pid = pi;
1208                 }
1209         }
1210
1211         return allowed_min_rxd;
1212 }
1213
1214 /*
1215  * Check input rxd is valid or not.
1216  * If input rxd is not greater than any of maximum number
1217  * of RXDs of every Rx queues and is not less than any of
1218  * minimal number of RXDs of every Rx queues, it is valid.
1219  * if valid, return 0, else return -1
1220  */
1221 int
1222 check_nb_rxd(queueid_t rxd)
1223 {
1224         uint16_t allowed_max_rxd;
1225         uint16_t allowed_min_rxd;
1226         portid_t pid = 0;
1227
1228         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1229         if (rxd > allowed_max_rxd) {
1230                 printf("Fail: input rxd (%u) can't be greater "
1231                        "than max_rxds (%u) of port %u\n",
1232                        rxd,
1233                        allowed_max_rxd,
1234                        pid);
1235                 return -1;
1236         }
1237
1238         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1239         if (rxd < allowed_min_rxd) {
1240                 printf("Fail: input rxd (%u) can't be less "
1241                        "than min_rxds (%u) of port %u\n",
1242                        rxd,
1243                        allowed_min_rxd,
1244                        pid);
1245                 return -1;
1246         }
1247
1248         return 0;
1249 }
1250
1251 /*
1252  * Get the allowed maximum number of TXDs of every rx queues.
1253  * *pid return the port id which has minimal value of
1254  * max_txd in every tx queue.
1255  */
1256 static uint16_t
1257 get_allowed_max_nb_txd(portid_t *pid)
1258 {
1259         uint16_t allowed_max_txd = UINT16_MAX;
1260         portid_t pi;
1261         struct rte_eth_dev_info dev_info;
1262
1263         RTE_ETH_FOREACH_DEV(pi) {
1264                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1265                         continue;
1266
1267                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1268                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1269                         *pid = pi;
1270                 }
1271         }
1272         return allowed_max_txd;
1273 }
1274
1275 /*
1276  * Get the allowed maximum number of TXDs of every tx queues.
1277  * *pid return the port id which has minimal value of
1278  * min_txd in every tx queue.
1279  */
1280 static uint16_t
1281 get_allowed_min_nb_txd(portid_t *pid)
1282 {
1283         uint16_t allowed_min_txd = 0;
1284         portid_t pi;
1285         struct rte_eth_dev_info dev_info;
1286
1287         RTE_ETH_FOREACH_DEV(pi) {
1288                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1289                         continue;
1290
1291                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1292                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1293                         *pid = pi;
1294                 }
1295         }
1296
1297         return allowed_min_txd;
1298 }
1299
1300 /*
1301  * Check input txd is valid or not.
1302  * If input txd is not greater than any of maximum number
1303  * of TXDs of every Rx queues, it is valid.
1304  * if valid, return 0, else return -1
1305  */
1306 int
1307 check_nb_txd(queueid_t txd)
1308 {
1309         uint16_t allowed_max_txd;
1310         uint16_t allowed_min_txd;
1311         portid_t pid = 0;
1312
1313         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1314         if (txd > allowed_max_txd) {
1315                 printf("Fail: input txd (%u) can't be greater "
1316                        "than max_txds (%u) of port %u\n",
1317                        txd,
1318                        allowed_max_txd,
1319                        pid);
1320                 return -1;
1321         }
1322
1323         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1324         if (txd < allowed_min_txd) {
1325                 printf("Fail: input txd (%u) can't be less "
1326                        "than min_txds (%u) of port %u\n",
1327                        txd,
1328                        allowed_min_txd,
1329                        pid);
1330                 return -1;
1331         }
1332         return 0;
1333 }
1334
1335
1336 /*
1337  * Get the allowed maximum number of hairpin queues.
1338  * *pid return the port id which has minimal value of
1339  * max_hairpin_queues in all ports.
1340  */
1341 queueid_t
1342 get_allowed_max_nb_hairpinq(portid_t *pid)
1343 {
1344         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1345         portid_t pi;
1346         struct rte_eth_hairpin_cap cap;
1347
1348         RTE_ETH_FOREACH_DEV(pi) {
1349                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1350                         *pid = pi;
1351                         return 0;
1352                 }
1353                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1354                         allowed_max_hairpinq = cap.max_nb_queues;
1355                         *pid = pi;
1356                 }
1357         }
1358         return allowed_max_hairpinq;
1359 }
1360
1361 /*
1362  * Check input hairpin is valid or not.
1363  * If input hairpin is not greater than any of maximum number
1364  * of hairpin queues of all ports, it is valid.
1365  * if valid, return 0, else return -1
1366  */
1367 int
1368 check_nb_hairpinq(queueid_t hairpinq)
1369 {
1370         queueid_t allowed_max_hairpinq;
1371         portid_t pid = 0;
1372
1373         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1374         if (hairpinq > allowed_max_hairpinq) {
1375                 printf("Fail: input hairpin (%u) can't be greater "
1376                        "than max_hairpin_queues (%u) of port %u\n",
1377                        hairpinq, allowed_max_hairpinq, pid);
1378                 return -1;
1379         }
1380         return 0;
1381 }
1382
1383 static void
1384 init_config(void)
1385 {
1386         portid_t pid;
1387         struct rte_port *port;
1388         struct rte_mempool *mbp;
1389         unsigned int nb_mbuf_per_pool;
1390         lcoreid_t  lc_id;
1391         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1392         struct rte_gro_param gro_param;
1393         uint32_t gso_types;
1394         uint16_t data_size;
1395         bool warning = 0;
1396         int k;
1397         int ret;
1398
1399         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1400
1401         /* Configuration of logical cores. */
1402         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1403                                 sizeof(struct fwd_lcore *) * nb_lcores,
1404                                 RTE_CACHE_LINE_SIZE);
1405         if (fwd_lcores == NULL) {
1406                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1407                                                         "failed\n", nb_lcores);
1408         }
1409         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1410                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1411                                                sizeof(struct fwd_lcore),
1412                                                RTE_CACHE_LINE_SIZE);
1413                 if (fwd_lcores[lc_id] == NULL) {
1414                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1415                                                                 "failed\n");
1416                 }
1417                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1418         }
1419
1420         RTE_ETH_FOREACH_DEV(pid) {
1421                 port = &ports[pid];
1422                 /* Apply default TxRx configuration for all ports */
1423                 port->dev_conf.txmode = tx_mode;
1424                 port->dev_conf.rxmode = rx_mode;
1425
1426                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1427                 if (ret != 0)
1428                         rte_exit(EXIT_FAILURE,
1429                                  "rte_eth_dev_info_get() failed\n");
1430
1431                 if (!(port->dev_info.tx_offload_capa &
1432                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1433                         port->dev_conf.txmode.offloads &=
1434                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1435                 if (numa_support) {
1436                         if (port_numa[pid] != NUMA_NO_CONFIG)
1437                                 port_per_socket[port_numa[pid]]++;
1438                         else {
1439                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1440
1441                                 /*
1442                                  * if socket_id is invalid,
1443                                  * set to the first available socket.
1444                                  */
1445                                 if (check_socket_id(socket_id) < 0)
1446                                         socket_id = socket_ids[0];
1447                                 port_per_socket[socket_id]++;
1448                         }
1449                 }
1450
1451                 /* Apply Rx offloads configuration */
1452                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1453                         port->rx_conf[k].offloads =
1454                                 port->dev_conf.rxmode.offloads;
1455                 /* Apply Tx offloads configuration */
1456                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1457                         port->tx_conf[k].offloads =
1458                                 port->dev_conf.txmode.offloads;
1459
1460                 /* set flag to initialize port/queue */
1461                 port->need_reconfig = 1;
1462                 port->need_reconfig_queues = 1;
1463                 port->tx_metadata = 0;
1464
1465                 /* Check for maximum number of segments per MTU. Accordingly
1466                  * update the mbuf data size.
1467                  */
1468                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1469                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1470                         data_size = rx_mode.max_rx_pkt_len /
1471                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1472
1473                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1474                                                         mbuf_data_size) {
1475                                 mbuf_data_size = data_size +
1476                                                  RTE_PKTMBUF_HEADROOM;
1477                                 warning = 1;
1478                         }
1479                 }
1480         }
1481
1482         if (warning)
1483                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1484                             mbuf_data_size);
1485
1486         /*
1487          * Create pools of mbuf.
1488          * If NUMA support is disabled, create a single pool of mbuf in
1489          * socket 0 memory by default.
1490          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1491          *
1492          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1493          * nb_txd can be configured at run time.
1494          */
1495         if (param_total_num_mbufs)
1496                 nb_mbuf_per_pool = param_total_num_mbufs;
1497         else {
1498                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1499                         (nb_lcores * mb_mempool_cache) +
1500                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1501                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1502         }
1503
1504         if (numa_support) {
1505                 uint8_t i;
1506
1507                 for (i = 0; i < num_sockets; i++)
1508                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1509                                                        nb_mbuf_per_pool,
1510                                                        socket_ids[i]);
1511         } else {
1512                 if (socket_num == UMA_NO_CONFIG)
1513                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1514                                                        nb_mbuf_per_pool, 0);
1515                 else
1516                         mempools[socket_num] = mbuf_pool_create
1517                                                         (mbuf_data_size,
1518                                                          nb_mbuf_per_pool,
1519                                                          socket_num);
1520         }
1521
1522         init_port_config();
1523
1524         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1525                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1526         /*
1527          * Records which Mbuf pool to use by each logical core, if needed.
1528          */
1529         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1530                 mbp = mbuf_pool_find(
1531                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1532
1533                 if (mbp == NULL)
1534                         mbp = mbuf_pool_find(0);
1535                 fwd_lcores[lc_id]->mbp = mbp;
1536                 /* initialize GSO context */
1537                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1538                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1539                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1540                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1541                         RTE_ETHER_CRC_LEN;
1542                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1543         }
1544
1545         /* Configuration of packet forwarding streams. */
1546         if (init_fwd_streams() < 0)
1547                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1548
1549         fwd_config_setup();
1550
1551         /* create a gro context for each lcore */
1552         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1553         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1554         gro_param.max_item_per_flow = MAX_PKT_BURST;
1555         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1556                 gro_param.socket_id = rte_lcore_to_socket_id(
1557                                 fwd_lcores_cpuids[lc_id]);
1558                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1559                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1560                         rte_exit(EXIT_FAILURE,
1561                                         "rte_gro_ctx_create() failed\n");
1562                 }
1563         }
1564
1565 #if defined RTE_LIBRTE_PMD_SOFTNIC
1566         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1567                 RTE_ETH_FOREACH_DEV(pid) {
1568                         port = &ports[pid];
1569                         const char *driver = port->dev_info.driver_name;
1570
1571                         if (strcmp(driver, "net_softnic") == 0)
1572                                 port->softport.fwd_lcore_arg = fwd_lcores;
1573                 }
1574         }
1575 #endif
1576
1577 }
1578
1579
1580 void
1581 reconfig(portid_t new_port_id, unsigned socket_id)
1582 {
1583         struct rte_port *port;
1584         int ret;
1585
1586         /* Reconfiguration of Ethernet ports. */
1587         port = &ports[new_port_id];
1588
1589         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1590         if (ret != 0)
1591                 return;
1592
1593         /* set flag to initialize port/queue */
1594         port->need_reconfig = 1;
1595         port->need_reconfig_queues = 1;
1596         port->socket_id = socket_id;
1597
1598         init_port_config();
1599 }
1600
1601
1602 int
1603 init_fwd_streams(void)
1604 {
1605         portid_t pid;
1606         struct rte_port *port;
1607         streamid_t sm_id, nb_fwd_streams_new;
1608         queueid_t q;
1609
1610         /* set socket id according to numa or not */
1611         RTE_ETH_FOREACH_DEV(pid) {
1612                 port = &ports[pid];
1613                 if (nb_rxq > port->dev_info.max_rx_queues) {
1614                         printf("Fail: nb_rxq(%d) is greater than "
1615                                 "max_rx_queues(%d)\n", nb_rxq,
1616                                 port->dev_info.max_rx_queues);
1617                         return -1;
1618                 }
1619                 if (nb_txq > port->dev_info.max_tx_queues) {
1620                         printf("Fail: nb_txq(%d) is greater than "
1621                                 "max_tx_queues(%d)\n", nb_txq,
1622                                 port->dev_info.max_tx_queues);
1623                         return -1;
1624                 }
1625                 if (numa_support) {
1626                         if (port_numa[pid] != NUMA_NO_CONFIG)
1627                                 port->socket_id = port_numa[pid];
1628                         else {
1629                                 port->socket_id = rte_eth_dev_socket_id(pid);
1630
1631                                 /*
1632                                  * if socket_id is invalid,
1633                                  * set to the first available socket.
1634                                  */
1635                                 if (check_socket_id(port->socket_id) < 0)
1636                                         port->socket_id = socket_ids[0];
1637                         }
1638                 }
1639                 else {
1640                         if (socket_num == UMA_NO_CONFIG)
1641                                 port->socket_id = 0;
1642                         else
1643                                 port->socket_id = socket_num;
1644                 }
1645         }
1646
1647         q = RTE_MAX(nb_rxq, nb_txq);
1648         if (q == 0) {
1649                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1650                 return -1;
1651         }
1652         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1653         if (nb_fwd_streams_new == nb_fwd_streams)
1654                 return 0;
1655         /* clear the old */
1656         if (fwd_streams != NULL) {
1657                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1658                         if (fwd_streams[sm_id] == NULL)
1659                                 continue;
1660                         rte_free(fwd_streams[sm_id]);
1661                         fwd_streams[sm_id] = NULL;
1662                 }
1663                 rte_free(fwd_streams);
1664                 fwd_streams = NULL;
1665         }
1666
1667         /* init new */
1668         nb_fwd_streams = nb_fwd_streams_new;
1669         if (nb_fwd_streams) {
1670                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1671                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1672                         RTE_CACHE_LINE_SIZE);
1673                 if (fwd_streams == NULL)
1674                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1675                                  " (struct fwd_stream *)) failed\n",
1676                                  nb_fwd_streams);
1677
1678                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1679                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1680                                 " struct fwd_stream", sizeof(struct fwd_stream),
1681                                 RTE_CACHE_LINE_SIZE);
1682                         if (fwd_streams[sm_id] == NULL)
1683                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1684                                          "(struct fwd_stream) failed\n");
1685                 }
1686         }
1687
1688         return 0;
1689 }
1690
1691 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1692 static void
1693 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1694 {
1695         unsigned int total_burst;
1696         unsigned int nb_burst;
1697         unsigned int burst_stats[3];
1698         uint16_t pktnb_stats[3];
1699         uint16_t nb_pkt;
1700         int burst_percent[3];
1701
1702         /*
1703          * First compute the total number of packet bursts and the
1704          * two highest numbers of bursts of the same number of packets.
1705          */
1706         total_burst = 0;
1707         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1708         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1709         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1710                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1711                 if (nb_burst == 0)
1712                         continue;
1713                 total_burst += nb_burst;
1714                 if (nb_burst > burst_stats[0]) {
1715                         burst_stats[1] = burst_stats[0];
1716                         pktnb_stats[1] = pktnb_stats[0];
1717                         burst_stats[0] = nb_burst;
1718                         pktnb_stats[0] = nb_pkt;
1719                 } else if (nb_burst > burst_stats[1]) {
1720                         burst_stats[1] = nb_burst;
1721                         pktnb_stats[1] = nb_pkt;
1722                 }
1723         }
1724         if (total_burst == 0)
1725                 return;
1726         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1727         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1728                burst_percent[0], (int) pktnb_stats[0]);
1729         if (burst_stats[0] == total_burst) {
1730                 printf("]\n");
1731                 return;
1732         }
1733         if (burst_stats[0] + burst_stats[1] == total_burst) {
1734                 printf(" + %d%% of %d pkts]\n",
1735                        100 - burst_percent[0], pktnb_stats[1]);
1736                 return;
1737         }
1738         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1739         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1740         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1741                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1742                 return;
1743         }
1744         printf(" + %d%% of %d pkts + %d%% of others]\n",
1745                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1746 }
1747 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1748
1749 static void
1750 fwd_stream_stats_display(streamid_t stream_id)
1751 {
1752         struct fwd_stream *fs;
1753         static const char *fwd_top_stats_border = "-------";
1754
1755         fs = fwd_streams[stream_id];
1756         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1757             (fs->fwd_dropped == 0))
1758                 return;
1759         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1760                "TX Port=%2d/Queue=%2d %s\n",
1761                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1762                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1763         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1764                " TX-dropped: %-14"PRIu64,
1765                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1766
1767         /* if checksum mode */
1768         if (cur_fwd_eng == &csum_fwd_engine) {
1769                 printf("  RX- bad IP checksum: %-14"PRIu64
1770                        "  Rx- bad L4 checksum: %-14"PRIu64
1771                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1772                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1773                         fs->rx_bad_outer_l4_csum);
1774         } else {
1775                 printf("\n");
1776         }
1777
1778 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1779         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1780         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1781 #endif
1782 }
1783
1784 void
1785 fwd_stats_display(void)
1786 {
1787         static const char *fwd_stats_border = "----------------------";
1788         static const char *acc_stats_border = "+++++++++++++++";
1789         struct {
1790                 struct fwd_stream *rx_stream;
1791                 struct fwd_stream *tx_stream;
1792                 uint64_t tx_dropped;
1793                 uint64_t rx_bad_ip_csum;
1794                 uint64_t rx_bad_l4_csum;
1795                 uint64_t rx_bad_outer_l4_csum;
1796         } ports_stats[RTE_MAX_ETHPORTS];
1797         uint64_t total_rx_dropped = 0;
1798         uint64_t total_tx_dropped = 0;
1799         uint64_t total_rx_nombuf = 0;
1800         struct rte_eth_stats stats;
1801 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1802         uint64_t fwd_cycles = 0;
1803 #endif
1804         uint64_t total_recv = 0;
1805         uint64_t total_xmit = 0;
1806         struct rte_port *port;
1807         streamid_t sm_id;
1808         portid_t pt_id;
1809         int i;
1810
1811         memset(ports_stats, 0, sizeof(ports_stats));
1812
1813         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1814                 struct fwd_stream *fs = fwd_streams[sm_id];
1815
1816                 if (cur_fwd_config.nb_fwd_streams >
1817                     cur_fwd_config.nb_fwd_ports) {
1818                         fwd_stream_stats_display(sm_id);
1819                 } else {
1820                         ports_stats[fs->tx_port].tx_stream = fs;
1821                         ports_stats[fs->rx_port].rx_stream = fs;
1822                 }
1823
1824                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1825
1826                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1827                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1828                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1829                                 fs->rx_bad_outer_l4_csum;
1830
1831 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1832                 fwd_cycles += fs->core_cycles;
1833 #endif
1834         }
1835         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1836                 uint8_t j;
1837
1838                 pt_id = fwd_ports_ids[i];
1839                 port = &ports[pt_id];
1840
1841                 rte_eth_stats_get(pt_id, &stats);
1842                 stats.ipackets -= port->stats.ipackets;
1843                 stats.opackets -= port->stats.opackets;
1844                 stats.ibytes -= port->stats.ibytes;
1845                 stats.obytes -= port->stats.obytes;
1846                 stats.imissed -= port->stats.imissed;
1847                 stats.oerrors -= port->stats.oerrors;
1848                 stats.rx_nombuf -= port->stats.rx_nombuf;
1849
1850                 total_recv += stats.ipackets;
1851                 total_xmit += stats.opackets;
1852                 total_rx_dropped += stats.imissed;
1853                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1854                 total_tx_dropped += stats.oerrors;
1855                 total_rx_nombuf  += stats.rx_nombuf;
1856
1857                 printf("\n  %s Forward statistics for port %-2d %s\n",
1858                        fwd_stats_border, pt_id, fwd_stats_border);
1859
1860                 if (!port->rx_queue_stats_mapping_enabled &&
1861                     !port->tx_queue_stats_mapping_enabled) {
1862                         printf("  RX-packets: %-14"PRIu64
1863                                " RX-dropped: %-14"PRIu64
1864                                "RX-total: %-"PRIu64"\n",
1865                                stats.ipackets, stats.imissed,
1866                                stats.ipackets + stats.imissed);
1867
1868                         if (cur_fwd_eng == &csum_fwd_engine)
1869                                 printf("  Bad-ipcsum: %-14"PRIu64
1870                                        " Bad-l4csum: %-14"PRIu64
1871                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1872                                        ports_stats[pt_id].rx_bad_ip_csum,
1873                                        ports_stats[pt_id].rx_bad_l4_csum,
1874                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1875                         if (stats.ierrors + stats.rx_nombuf > 0) {
1876                                 printf("  RX-error: %-"PRIu64"\n",
1877                                        stats.ierrors);
1878                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1879                                        stats.rx_nombuf);
1880                         }
1881
1882                         printf("  TX-packets: %-14"PRIu64
1883                                " TX-dropped: %-14"PRIu64
1884                                "TX-total: %-"PRIu64"\n",
1885                                stats.opackets, ports_stats[pt_id].tx_dropped,
1886                                stats.opackets + ports_stats[pt_id].tx_dropped);
1887                 } else {
1888                         printf("  RX-packets:             %14"PRIu64
1889                                "    RX-dropped:%14"PRIu64
1890                                "    RX-total:%14"PRIu64"\n",
1891                                stats.ipackets, stats.imissed,
1892                                stats.ipackets + stats.imissed);
1893
1894                         if (cur_fwd_eng == &csum_fwd_engine)
1895                                 printf("  Bad-ipcsum:%14"PRIu64
1896                                        "    Bad-l4csum:%14"PRIu64
1897                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1898                                        ports_stats[pt_id].rx_bad_ip_csum,
1899                                        ports_stats[pt_id].rx_bad_l4_csum,
1900                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1901                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1902                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1903                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1904                                        stats.rx_nombuf);
1905                         }
1906
1907                         printf("  TX-packets:             %14"PRIu64
1908                                "    TX-dropped:%14"PRIu64
1909                                "    TX-total:%14"PRIu64"\n",
1910                                stats.opackets, ports_stats[pt_id].tx_dropped,
1911                                stats.opackets + ports_stats[pt_id].tx_dropped);
1912                 }
1913
1914 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1915                 if (ports_stats[pt_id].rx_stream)
1916                         pkt_burst_stats_display("RX",
1917                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1918                 if (ports_stats[pt_id].tx_stream)
1919                         pkt_burst_stats_display("TX",
1920                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1921 #endif
1922
1923                 if (port->rx_queue_stats_mapping_enabled) {
1924                         printf("\n");
1925                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1926                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1927                                        "     RX-errors:%14"PRIu64
1928                                        "    RX-bytes:%14"PRIu64"\n",
1929                                        j, stats.q_ipackets[j],
1930                                        stats.q_errors[j], stats.q_ibytes[j]);
1931                         }
1932                         printf("\n");
1933                 }
1934                 if (port->tx_queue_stats_mapping_enabled) {
1935                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1936                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1937                                        "                                 TX-bytes:%14"
1938                                        PRIu64"\n",
1939                                        j, stats.q_opackets[j],
1940                                        stats.q_obytes[j]);
1941                         }
1942                 }
1943
1944                 printf("  %s--------------------------------%s\n",
1945                        fwd_stats_border, fwd_stats_border);
1946         }
1947
1948         printf("\n  %s Accumulated forward statistics for all ports"
1949                "%s\n",
1950                acc_stats_border, acc_stats_border);
1951         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1952                "%-"PRIu64"\n"
1953                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1954                "%-"PRIu64"\n",
1955                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1956                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1957         if (total_rx_nombuf > 0)
1958                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1959         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1960                "%s\n",
1961                acc_stats_border, acc_stats_border);
1962 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1963 #define CYC_PER_MHZ 1E6
1964         if (total_recv > 0)
1965                 printf("\n  CPU cycles/packet=%u (total cycles="
1966                        "%"PRIu64" / total RX packets=%"PRIu64") at %"PRIu64
1967                        " MHz Clock\n",
1968                        (unsigned int)(fwd_cycles / total_recv),
1969                        fwd_cycles, total_recv,
1970                        (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1971 #endif
1972 }
1973
1974 void
1975 fwd_stats_reset(void)
1976 {
1977         streamid_t sm_id;
1978         portid_t pt_id;
1979         int i;
1980
1981         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1982                 pt_id = fwd_ports_ids[i];
1983                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1984         }
1985         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1986                 struct fwd_stream *fs = fwd_streams[sm_id];
1987
1988                 fs->rx_packets = 0;
1989                 fs->tx_packets = 0;
1990                 fs->fwd_dropped = 0;
1991                 fs->rx_bad_ip_csum = 0;
1992                 fs->rx_bad_l4_csum = 0;
1993                 fs->rx_bad_outer_l4_csum = 0;
1994
1995 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1996                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1997                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1998 #endif
1999 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
2000                 fs->core_cycles = 0;
2001 #endif
2002         }
2003 }
2004
2005 static void
2006 flush_fwd_rx_queues(void)
2007 {
2008         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2009         portid_t  rxp;
2010         portid_t port_id;
2011         queueid_t rxq;
2012         uint16_t  nb_rx;
2013         uint16_t  i;
2014         uint8_t   j;
2015         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2016         uint64_t timer_period;
2017
2018         /* convert to number of cycles */
2019         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2020
2021         for (j = 0; j < 2; j++) {
2022                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2023                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2024                                 port_id = fwd_ports_ids[rxp];
2025                                 /**
2026                                 * testpmd can stuck in the below do while loop
2027                                 * if rte_eth_rx_burst() always returns nonzero
2028                                 * packets. So timer is added to exit this loop
2029                                 * after 1sec timer expiry.
2030                                 */
2031                                 prev_tsc = rte_rdtsc();
2032                                 do {
2033                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2034                                                 pkts_burst, MAX_PKT_BURST);
2035                                         for (i = 0; i < nb_rx; i++)
2036                                                 rte_pktmbuf_free(pkts_burst[i]);
2037
2038                                         cur_tsc = rte_rdtsc();
2039                                         diff_tsc = cur_tsc - prev_tsc;
2040                                         timer_tsc += diff_tsc;
2041                                 } while ((nb_rx > 0) &&
2042                                         (timer_tsc < timer_period));
2043                                 timer_tsc = 0;
2044                         }
2045                 }
2046                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2047         }
2048 }
2049
2050 static void
2051 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2052 {
2053         struct fwd_stream **fsm;
2054         streamid_t nb_fs;
2055         streamid_t sm_id;
2056 #ifdef RTE_LIBRTE_BITRATE
2057         uint64_t tics_per_1sec;
2058         uint64_t tics_datum;
2059         uint64_t tics_current;
2060         uint16_t i, cnt_ports;
2061
2062         cnt_ports = nb_ports;
2063         tics_datum = rte_rdtsc();
2064         tics_per_1sec = rte_get_timer_hz();
2065 #endif
2066         fsm = &fwd_streams[fc->stream_idx];
2067         nb_fs = fc->stream_nb;
2068         do {
2069                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2070                         (*pkt_fwd)(fsm[sm_id]);
2071 #ifdef RTE_LIBRTE_BITRATE
2072                 if (bitrate_enabled != 0 &&
2073                                 bitrate_lcore_id == rte_lcore_id()) {
2074                         tics_current = rte_rdtsc();
2075                         if (tics_current - tics_datum >= tics_per_1sec) {
2076                                 /* Periodic bitrate calculation */
2077                                 for (i = 0; i < cnt_ports; i++)
2078                                         rte_stats_bitrate_calc(bitrate_data,
2079                                                 ports_ids[i]);
2080                                 tics_datum = tics_current;
2081                         }
2082                 }
2083 #endif
2084 #ifdef RTE_LIBRTE_LATENCY_STATS
2085                 if (latencystats_enabled != 0 &&
2086                                 latencystats_lcore_id == rte_lcore_id())
2087                         rte_latencystats_update();
2088 #endif
2089
2090         } while (! fc->stopped);
2091 }
2092
2093 static int
2094 start_pkt_forward_on_core(void *fwd_arg)
2095 {
2096         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2097                              cur_fwd_config.fwd_eng->packet_fwd);
2098         return 0;
2099 }
2100
2101 /*
2102  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2103  * Used to start communication flows in network loopback test configurations.
2104  */
2105 static int
2106 run_one_txonly_burst_on_core(void *fwd_arg)
2107 {
2108         struct fwd_lcore *fwd_lc;
2109         struct fwd_lcore tmp_lcore;
2110
2111         fwd_lc = (struct fwd_lcore *) fwd_arg;
2112         tmp_lcore = *fwd_lc;
2113         tmp_lcore.stopped = 1;
2114         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2115         return 0;
2116 }
2117
2118 /*
2119  * Launch packet forwarding:
2120  *     - Setup per-port forwarding context.
2121  *     - launch logical cores with their forwarding configuration.
2122  */
2123 static void
2124 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2125 {
2126         port_fwd_begin_t port_fwd_begin;
2127         unsigned int i;
2128         unsigned int lc_id;
2129         int diag;
2130
2131         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2132         if (port_fwd_begin != NULL) {
2133                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2134                         (*port_fwd_begin)(fwd_ports_ids[i]);
2135         }
2136         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2137                 lc_id = fwd_lcores_cpuids[i];
2138                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2139                         fwd_lcores[i]->stopped = 0;
2140                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2141                                                      fwd_lcores[i], lc_id);
2142                         if (diag != 0)
2143                                 printf("launch lcore %u failed - diag=%d\n",
2144                                        lc_id, diag);
2145                 }
2146         }
2147 }
2148
2149 /*
2150  * Launch packet forwarding configuration.
2151  */
2152 void
2153 start_packet_forwarding(int with_tx_first)
2154 {
2155         port_fwd_begin_t port_fwd_begin;
2156         port_fwd_end_t  port_fwd_end;
2157         struct rte_port *port;
2158         unsigned int i;
2159         portid_t   pt_id;
2160
2161         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2162                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2163
2164         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2165                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2166
2167         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2168                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2169                 (!nb_rxq || !nb_txq))
2170                 rte_exit(EXIT_FAILURE,
2171                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2172                         cur_fwd_eng->fwd_mode_name);
2173
2174         if (all_ports_started() == 0) {
2175                 printf("Not all ports were started\n");
2176                 return;
2177         }
2178         if (test_done == 0) {
2179                 printf("Packet forwarding already started\n");
2180                 return;
2181         }
2182
2183
2184         if(dcb_test) {
2185                 for (i = 0; i < nb_fwd_ports; i++) {
2186                         pt_id = fwd_ports_ids[i];
2187                         port = &ports[pt_id];
2188                         if (!port->dcb_flag) {
2189                                 printf("In DCB mode, all forwarding ports must "
2190                                        "be configured in this mode.\n");
2191                                 return;
2192                         }
2193                 }
2194                 if (nb_fwd_lcores == 1) {
2195                         printf("In DCB mode,the nb forwarding cores "
2196                                "should be larger than 1.\n");
2197                         return;
2198                 }
2199         }
2200         test_done = 0;
2201
2202         fwd_config_setup();
2203
2204         if(!no_flush_rx)
2205                 flush_fwd_rx_queues();
2206
2207         pkt_fwd_config_display(&cur_fwd_config);
2208         rxtx_config_display();
2209
2210         fwd_stats_reset();
2211         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2212                 pt_id = fwd_ports_ids[i];
2213                 port = &ports[pt_id];
2214                 map_port_queue_stats_mapping_registers(pt_id, port);
2215         }
2216         if (with_tx_first) {
2217                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2218                 if (port_fwd_begin != NULL) {
2219                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2220                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2221                 }
2222                 while (with_tx_first--) {
2223                         launch_packet_forwarding(
2224                                         run_one_txonly_burst_on_core);
2225                         rte_eal_mp_wait_lcore();
2226                 }
2227                 port_fwd_end = tx_only_engine.port_fwd_end;
2228                 if (port_fwd_end != NULL) {
2229                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2230                                 (*port_fwd_end)(fwd_ports_ids[i]);
2231                 }
2232         }
2233         launch_packet_forwarding(start_pkt_forward_on_core);
2234 }
2235
2236 void
2237 stop_packet_forwarding(void)
2238 {
2239         port_fwd_end_t port_fwd_end;
2240         lcoreid_t lc_id;
2241         portid_t pt_id;
2242         int i;
2243
2244         if (test_done) {
2245                 printf("Packet forwarding not started\n");
2246                 return;
2247         }
2248         printf("Telling cores to stop...");
2249         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2250                 fwd_lcores[lc_id]->stopped = 1;
2251         printf("\nWaiting for lcores to finish...\n");
2252         rte_eal_mp_wait_lcore();
2253         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2254         if (port_fwd_end != NULL) {
2255                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2256                         pt_id = fwd_ports_ids[i];
2257                         (*port_fwd_end)(pt_id);
2258                 }
2259         }
2260
2261         fwd_stats_display();
2262
2263         printf("\nDone.\n");
2264         test_done = 1;
2265 }
2266
2267 void
2268 dev_set_link_up(portid_t pid)
2269 {
2270         if (rte_eth_dev_set_link_up(pid) < 0)
2271                 printf("\nSet link up fail.\n");
2272 }
2273
2274 void
2275 dev_set_link_down(portid_t pid)
2276 {
2277         if (rte_eth_dev_set_link_down(pid) < 0)
2278                 printf("\nSet link down fail.\n");
2279 }
2280
2281 static int
2282 all_ports_started(void)
2283 {
2284         portid_t pi;
2285         struct rte_port *port;
2286
2287         RTE_ETH_FOREACH_DEV(pi) {
2288                 port = &ports[pi];
2289                 /* Check if there is a port which is not started */
2290                 if ((port->port_status != RTE_PORT_STARTED) &&
2291                         (port->slave_flag == 0))
2292                         return 0;
2293         }
2294
2295         /* No port is not started */
2296         return 1;
2297 }
2298
2299 int
2300 port_is_stopped(portid_t port_id)
2301 {
2302         struct rte_port *port = &ports[port_id];
2303
2304         if ((port->port_status != RTE_PORT_STOPPED) &&
2305             (port->slave_flag == 0))
2306                 return 0;
2307         return 1;
2308 }
2309
2310 int
2311 all_ports_stopped(void)
2312 {
2313         portid_t pi;
2314
2315         RTE_ETH_FOREACH_DEV(pi) {
2316                 if (!port_is_stopped(pi))
2317                         return 0;
2318         }
2319
2320         return 1;
2321 }
2322
2323 int
2324 port_is_started(portid_t port_id)
2325 {
2326         if (port_id_is_invalid(port_id, ENABLED_WARN))
2327                 return 0;
2328
2329         if (ports[port_id].port_status != RTE_PORT_STARTED)
2330                 return 0;
2331
2332         return 1;
2333 }
2334
2335 /* Configure the Rx and Tx hairpin queues for the selected port. */
2336 static int
2337 setup_hairpin_queues(portid_t pi)
2338 {
2339         queueid_t qi;
2340         struct rte_eth_hairpin_conf hairpin_conf = {
2341                 .peer_count = 1,
2342         };
2343         int i;
2344         int diag;
2345         struct rte_port *port = &ports[pi];
2346
2347         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2348                 hairpin_conf.peers[0].port = pi;
2349                 hairpin_conf.peers[0].queue = i + nb_rxq;
2350                 diag = rte_eth_tx_hairpin_queue_setup
2351                         (pi, qi, nb_txd, &hairpin_conf);
2352                 i++;
2353                 if (diag == 0)
2354                         continue;
2355
2356                 /* Fail to setup rx queue, return */
2357                 if (rte_atomic16_cmpset(&(port->port_status),
2358                                         RTE_PORT_HANDLING,
2359                                         RTE_PORT_STOPPED) == 0)
2360                         printf("Port %d can not be set back "
2361                                         "to stopped\n", pi);
2362                 printf("Fail to configure port %d hairpin "
2363                                 "queues\n", pi);
2364                 /* try to reconfigure queues next time */
2365                 port->need_reconfig_queues = 1;
2366                 return -1;
2367         }
2368         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2369                 hairpin_conf.peers[0].port = pi;
2370                 hairpin_conf.peers[0].queue = i + nb_txq;
2371                 diag = rte_eth_rx_hairpin_queue_setup
2372                         (pi, qi, nb_rxd, &hairpin_conf);
2373                 i++;
2374                 if (diag == 0)
2375                         continue;
2376
2377                 /* Fail to setup rx queue, return */
2378                 if (rte_atomic16_cmpset(&(port->port_status),
2379                                         RTE_PORT_HANDLING,
2380                                         RTE_PORT_STOPPED) == 0)
2381                         printf("Port %d can not be set back "
2382                                         "to stopped\n", pi);
2383                 printf("Fail to configure port %d hairpin "
2384                                 "queues\n", pi);
2385                 /* try to reconfigure queues next time */
2386                 port->need_reconfig_queues = 1;
2387                 return -1;
2388         }
2389         return 0;
2390 }
2391
2392 int
2393 start_port(portid_t pid)
2394 {
2395         int diag, need_check_link_status = -1;
2396         portid_t pi;
2397         queueid_t qi;
2398         struct rte_port *port;
2399         struct rte_ether_addr mac_addr;
2400         struct rte_eth_hairpin_cap cap;
2401
2402         if (port_id_is_invalid(pid, ENABLED_WARN))
2403                 return 0;
2404
2405         if(dcb_config)
2406                 dcb_test = 1;
2407         RTE_ETH_FOREACH_DEV(pi) {
2408                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2409                         continue;
2410
2411                 need_check_link_status = 0;
2412                 port = &ports[pi];
2413                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2414                                                  RTE_PORT_HANDLING) == 0) {
2415                         printf("Port %d is now not stopped\n", pi);
2416                         continue;
2417                 }
2418
2419                 if (port->need_reconfig > 0) {
2420                         port->need_reconfig = 0;
2421
2422                         if (flow_isolate_all) {
2423                                 int ret = port_flow_isolate(pi, 1);
2424                                 if (ret) {
2425                                         printf("Failed to apply isolated"
2426                                                " mode on port %d\n", pi);
2427                                         return -1;
2428                                 }
2429                         }
2430                         configure_rxtx_dump_callbacks(0);
2431                         printf("Configuring Port %d (socket %u)\n", pi,
2432                                         port->socket_id);
2433                         if (nb_hairpinq > 0 &&
2434                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2435                                 printf("Port %d doesn't support hairpin "
2436                                        "queues\n", pi);
2437                                 return -1;
2438                         }
2439                         /* configure port */
2440                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2441                                                      nb_txq + nb_hairpinq,
2442                                                      &(port->dev_conf));
2443                         if (diag != 0) {
2444                                 if (rte_atomic16_cmpset(&(port->port_status),
2445                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2446                                         printf("Port %d can not be set back "
2447                                                         "to stopped\n", pi);
2448                                 printf("Fail to configure port %d\n", pi);
2449                                 /* try to reconfigure port next time */
2450                                 port->need_reconfig = 1;
2451                                 return -1;
2452                         }
2453                 }
2454                 if (port->need_reconfig_queues > 0) {
2455                         port->need_reconfig_queues = 0;
2456                         /* setup tx queues */
2457                         for (qi = 0; qi < nb_txq; qi++) {
2458                                 if ((numa_support) &&
2459                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2460                                         diag = rte_eth_tx_queue_setup(pi, qi,
2461                                                 port->nb_tx_desc[qi],
2462                                                 txring_numa[pi],
2463                                                 &(port->tx_conf[qi]));
2464                                 else
2465                                         diag = rte_eth_tx_queue_setup(pi, qi,
2466                                                 port->nb_tx_desc[qi],
2467                                                 port->socket_id,
2468                                                 &(port->tx_conf[qi]));
2469
2470                                 if (diag == 0)
2471                                         continue;
2472
2473                                 /* Fail to setup tx queue, return */
2474                                 if (rte_atomic16_cmpset(&(port->port_status),
2475                                                         RTE_PORT_HANDLING,
2476                                                         RTE_PORT_STOPPED) == 0)
2477                                         printf("Port %d can not be set back "
2478                                                         "to stopped\n", pi);
2479                                 printf("Fail to configure port %d tx queues\n",
2480                                        pi);
2481                                 /* try to reconfigure queues next time */
2482                                 port->need_reconfig_queues = 1;
2483                                 return -1;
2484                         }
2485                         for (qi = 0; qi < nb_rxq; qi++) {
2486                                 /* setup rx queues */
2487                                 if ((numa_support) &&
2488                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2489                                         struct rte_mempool * mp =
2490                                                 mbuf_pool_find(rxring_numa[pi]);
2491                                         if (mp == NULL) {
2492                                                 printf("Failed to setup RX queue:"
2493                                                         "No mempool allocation"
2494                                                         " on the socket %d\n",
2495                                                         rxring_numa[pi]);
2496                                                 return -1;
2497                                         }
2498
2499                                         diag = rte_eth_rx_queue_setup(pi, qi,
2500                                              port->nb_rx_desc[qi],
2501                                              rxring_numa[pi],
2502                                              &(port->rx_conf[qi]),
2503                                              mp);
2504                                 } else {
2505                                         struct rte_mempool *mp =
2506                                                 mbuf_pool_find(port->socket_id);
2507                                         if (mp == NULL) {
2508                                                 printf("Failed to setup RX queue:"
2509                                                         "No mempool allocation"
2510                                                         " on the socket %d\n",
2511                                                         port->socket_id);
2512                                                 return -1;
2513                                         }
2514                                         diag = rte_eth_rx_queue_setup(pi, qi,
2515                                              port->nb_rx_desc[qi],
2516                                              port->socket_id,
2517                                              &(port->rx_conf[qi]),
2518                                              mp);
2519                                 }
2520                                 if (diag == 0)
2521                                         continue;
2522
2523                                 /* Fail to setup rx queue, return */
2524                                 if (rte_atomic16_cmpset(&(port->port_status),
2525                                                         RTE_PORT_HANDLING,
2526                                                         RTE_PORT_STOPPED) == 0)
2527                                         printf("Port %d can not be set back "
2528                                                         "to stopped\n", pi);
2529                                 printf("Fail to configure port %d rx queues\n",
2530                                        pi);
2531                                 /* try to reconfigure queues next time */
2532                                 port->need_reconfig_queues = 1;
2533                                 return -1;
2534                         }
2535                         /* setup hairpin queues */
2536                         if (setup_hairpin_queues(pi) != 0)
2537                                 return -1;
2538                 }
2539                 configure_rxtx_dump_callbacks(verbose_level);
2540                 if (clear_ptypes) {
2541                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2542                                         NULL, 0);
2543                         if (diag < 0)
2544                                 printf(
2545                                 "Port %d: Failed to disable Ptype parsing\n",
2546                                 pi);
2547                 }
2548
2549                 /* start port */
2550                 if (rte_eth_dev_start(pi) < 0) {
2551                         printf("Fail to start port %d\n", pi);
2552
2553                         /* Fail to setup rx queue, return */
2554                         if (rte_atomic16_cmpset(&(port->port_status),
2555                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2556                                 printf("Port %d can not be set back to "
2557                                                         "stopped\n", pi);
2558                         continue;
2559                 }
2560
2561                 if (rte_atomic16_cmpset(&(port->port_status),
2562                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2563                         printf("Port %d can not be set into started\n", pi);
2564
2565                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2566                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2567                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2568                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2569                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2570
2571                 /* at least one port started, need checking link status */
2572                 need_check_link_status = 1;
2573         }
2574
2575         if (need_check_link_status == 1 && !no_link_check)
2576                 check_all_ports_link_status(RTE_PORT_ALL);
2577         else if (need_check_link_status == 0)
2578                 printf("Please stop the ports first\n");
2579
2580         printf("Done\n");
2581         return 0;
2582 }
2583
2584 void
2585 stop_port(portid_t pid)
2586 {
2587         portid_t pi;
2588         struct rte_port *port;
2589         int need_check_link_status = 0;
2590
2591         if (dcb_test) {
2592                 dcb_test = 0;
2593                 dcb_config = 0;
2594         }
2595
2596         if (port_id_is_invalid(pid, ENABLED_WARN))
2597                 return;
2598
2599         printf("Stopping ports...\n");
2600
2601         RTE_ETH_FOREACH_DEV(pi) {
2602                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2603                         continue;
2604
2605                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2606                         printf("Please remove port %d from forwarding configuration.\n", pi);
2607                         continue;
2608                 }
2609
2610                 if (port_is_bonding_slave(pi)) {
2611                         printf("Please remove port %d from bonded device.\n", pi);
2612                         continue;
2613                 }
2614
2615                 port = &ports[pi];
2616                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2617                                                 RTE_PORT_HANDLING) == 0)
2618                         continue;
2619
2620                 rte_eth_dev_stop(pi);
2621
2622                 if (rte_atomic16_cmpset(&(port->port_status),
2623                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2624                         printf("Port %d can not be set into stopped\n", pi);
2625                 need_check_link_status = 1;
2626         }
2627         if (need_check_link_status && !no_link_check)
2628                 check_all_ports_link_status(RTE_PORT_ALL);
2629
2630         printf("Done\n");
2631 }
2632
2633 static void
2634 remove_invalid_ports_in(portid_t *array, portid_t *total)
2635 {
2636         portid_t i;
2637         portid_t new_total = 0;
2638
2639         for (i = 0; i < *total; i++)
2640                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2641                         array[new_total] = array[i];
2642                         new_total++;
2643                 }
2644         *total = new_total;
2645 }
2646
2647 static void
2648 remove_invalid_ports(void)
2649 {
2650         remove_invalid_ports_in(ports_ids, &nb_ports);
2651         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2652         nb_cfg_ports = nb_fwd_ports;
2653 }
2654
2655 void
2656 close_port(portid_t pid)
2657 {
2658         portid_t pi;
2659         struct rte_port *port;
2660
2661         if (port_id_is_invalid(pid, ENABLED_WARN))
2662                 return;
2663
2664         printf("Closing ports...\n");
2665
2666         RTE_ETH_FOREACH_DEV(pi) {
2667                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2668                         continue;
2669
2670                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2671                         printf("Please remove port %d from forwarding configuration.\n", pi);
2672                         continue;
2673                 }
2674
2675                 if (port_is_bonding_slave(pi)) {
2676                         printf("Please remove port %d from bonded device.\n", pi);
2677                         continue;
2678                 }
2679
2680                 port = &ports[pi];
2681                 if (rte_atomic16_cmpset(&(port->port_status),
2682                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2683                         printf("Port %d is already closed\n", pi);
2684                         continue;
2685                 }
2686
2687                 if (rte_atomic16_cmpset(&(port->port_status),
2688                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2689                         printf("Port %d is now not stopped\n", pi);
2690                         continue;
2691                 }
2692
2693                 if (port->flow_list)
2694                         port_flow_flush(pi);
2695                 rte_eth_dev_close(pi);
2696
2697                 remove_invalid_ports();
2698
2699                 if (rte_atomic16_cmpset(&(port->port_status),
2700                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2701                         printf("Port %d cannot be set to closed\n", pi);
2702         }
2703
2704         printf("Done\n");
2705 }
2706
2707 void
2708 reset_port(portid_t pid)
2709 {
2710         int diag;
2711         portid_t pi;
2712         struct rte_port *port;
2713
2714         if (port_id_is_invalid(pid, ENABLED_WARN))
2715                 return;
2716
2717         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2718                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2719                 printf("Can not reset port(s), please stop port(s) first.\n");
2720                 return;
2721         }
2722
2723         printf("Resetting ports...\n");
2724
2725         RTE_ETH_FOREACH_DEV(pi) {
2726                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2727                         continue;
2728
2729                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2730                         printf("Please remove port %d from forwarding "
2731                                "configuration.\n", pi);
2732                         continue;
2733                 }
2734
2735                 if (port_is_bonding_slave(pi)) {
2736                         printf("Please remove port %d from bonded device.\n",
2737                                pi);
2738                         continue;
2739                 }
2740
2741                 diag = rte_eth_dev_reset(pi);
2742                 if (diag == 0) {
2743                         port = &ports[pi];
2744                         port->need_reconfig = 1;
2745                         port->need_reconfig_queues = 1;
2746                 } else {
2747                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2748                 }
2749         }
2750
2751         printf("Done\n");
2752 }
2753
2754 void
2755 attach_port(char *identifier)
2756 {
2757         portid_t pi;
2758         struct rte_dev_iterator iterator;
2759
2760         printf("Attaching a new port...\n");
2761
2762         if (identifier == NULL) {
2763                 printf("Invalid parameters are specified\n");
2764                 return;
2765         }
2766
2767         if (rte_dev_probe(identifier) < 0) {
2768                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2769                 return;
2770         }
2771
2772         /* first attach mode: event */
2773         if (setup_on_probe_event) {
2774                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2775                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2776                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2777                                         ports[pi].need_setup != 0)
2778                                 setup_attached_port(pi);
2779                 return;
2780         }
2781
2782         /* second attach mode: iterator */
2783         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2784                 /* setup ports matching the devargs used for probing */
2785                 if (port_is_forwarding(pi))
2786                         continue; /* port was already attached before */
2787                 setup_attached_port(pi);
2788         }
2789 }
2790
2791 static void
2792 setup_attached_port(portid_t pi)
2793 {
2794         unsigned int socket_id;
2795         int ret;
2796
2797         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2798         /* if socket_id is invalid, set to the first available socket. */
2799         if (check_socket_id(socket_id) < 0)
2800                 socket_id = socket_ids[0];
2801         reconfig(pi, socket_id);
2802         ret = rte_eth_promiscuous_enable(pi);
2803         if (ret != 0)
2804                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2805                         pi, rte_strerror(-ret));
2806
2807         ports_ids[nb_ports++] = pi;
2808         fwd_ports_ids[nb_fwd_ports++] = pi;
2809         nb_cfg_ports = nb_fwd_ports;
2810         ports[pi].need_setup = 0;
2811         ports[pi].port_status = RTE_PORT_STOPPED;
2812
2813         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2814         printf("Done\n");
2815 }
2816
2817 static void
2818 detach_device(struct rte_device *dev)
2819 {
2820         portid_t sibling;
2821
2822         if (dev == NULL) {
2823                 printf("Device already removed\n");
2824                 return;
2825         }
2826
2827         printf("Removing a device...\n");
2828
2829         if (rte_dev_remove(dev) < 0) {
2830                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2831                 return;
2832         }
2833         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2834                 /* reset mapping between old ports and removed device */
2835                 rte_eth_devices[sibling].device = NULL;
2836                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2837                         /* sibling ports are forced to be closed */
2838                         ports[sibling].port_status = RTE_PORT_CLOSED;
2839                         printf("Port %u is closed\n", sibling);
2840                 }
2841         }
2842
2843         remove_invalid_ports();
2844
2845         printf("Device is detached\n");
2846         printf("Now total ports is %d\n", nb_ports);
2847         printf("Done\n");
2848         return;
2849 }
2850
2851 void
2852 detach_port_device(portid_t port_id)
2853 {
2854         if (port_id_is_invalid(port_id, ENABLED_WARN))
2855                 return;
2856
2857         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2858                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2859                         printf("Port not stopped\n");
2860                         return;
2861                 }
2862                 printf("Port was not closed\n");
2863                 if (ports[port_id].flow_list)
2864                         port_flow_flush(port_id);
2865         }
2866
2867         detach_device(rte_eth_devices[port_id].device);
2868 }
2869
2870 void
2871 detach_devargs(char *identifier)
2872 {
2873         struct rte_dev_iterator iterator;
2874         struct rte_devargs da;
2875         portid_t port_id;
2876
2877         printf("Removing a device...\n");
2878
2879         memset(&da, 0, sizeof(da));
2880         if (rte_devargs_parsef(&da, "%s", identifier)) {
2881                 printf("cannot parse identifier\n");
2882                 if (da.args)
2883                         free(da.args);
2884                 return;
2885         }
2886
2887         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2888                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2889                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2890                                 printf("Port %u not stopped\n", port_id);
2891                                 rte_eth_iterator_cleanup(&iterator);
2892                                 return;
2893                         }
2894
2895                         /* sibling ports are forced to be closed */
2896                         if (ports[port_id].flow_list)
2897                                 port_flow_flush(port_id);
2898                         ports[port_id].port_status = RTE_PORT_CLOSED;
2899                         printf("Port %u is now closed\n", port_id);
2900                 }
2901         }
2902
2903         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2904                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2905                             da.name, da.bus->name);
2906                 return;
2907         }
2908
2909         remove_invalid_ports();
2910
2911         printf("Device %s is detached\n", identifier);
2912         printf("Now total ports is %d\n", nb_ports);
2913         printf("Done\n");
2914 }
2915
2916 void
2917 pmd_test_exit(void)
2918 {
2919         portid_t pt_id;
2920         int ret;
2921         int i;
2922
2923         if (test_done == 0)
2924                 stop_packet_forwarding();
2925
2926         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2927                 if (mempools[i]) {
2928                         if (mp_alloc_type == MP_ALLOC_ANON)
2929                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2930                                                      NULL);
2931                 }
2932         }
2933         if (ports != NULL) {
2934                 no_link_check = 1;
2935                 RTE_ETH_FOREACH_DEV(pt_id) {
2936                         printf("\nStopping port %d...\n", pt_id);
2937                         fflush(stdout);
2938                         stop_port(pt_id);
2939                 }
2940                 RTE_ETH_FOREACH_DEV(pt_id) {
2941                         printf("\nShutting down port %d...\n", pt_id);
2942                         fflush(stdout);
2943                         close_port(pt_id);
2944                 }
2945         }
2946
2947         if (hot_plug) {
2948                 ret = rte_dev_event_monitor_stop();
2949                 if (ret) {
2950                         RTE_LOG(ERR, EAL,
2951                                 "fail to stop device event monitor.");
2952                         return;
2953                 }
2954
2955                 ret = rte_dev_event_callback_unregister(NULL,
2956                         dev_event_callback, NULL);
2957                 if (ret < 0) {
2958                         RTE_LOG(ERR, EAL,
2959                                 "fail to unregister device event callback.\n");
2960                         return;
2961                 }
2962
2963                 ret = rte_dev_hotplug_handle_disable();
2964                 if (ret) {
2965                         RTE_LOG(ERR, EAL,
2966                                 "fail to disable hotplug handling.\n");
2967                         return;
2968                 }
2969         }
2970         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2971                 if (mempools[i])
2972                         rte_mempool_free(mempools[i]);
2973         }
2974
2975         printf("\nBye...\n");
2976 }
2977
2978 typedef void (*cmd_func_t)(void);
2979 struct pmd_test_command {
2980         const char *cmd_name;
2981         cmd_func_t cmd_func;
2982 };
2983
2984 /* Check the link status of all ports in up to 9s, and print them finally */
2985 static void
2986 check_all_ports_link_status(uint32_t port_mask)
2987 {
2988 #define CHECK_INTERVAL 100 /* 100ms */
2989 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2990         portid_t portid;
2991         uint8_t count, all_ports_up, print_flag = 0;
2992         struct rte_eth_link link;
2993         int ret;
2994
2995         printf("Checking link statuses...\n");
2996         fflush(stdout);
2997         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2998                 all_ports_up = 1;
2999                 RTE_ETH_FOREACH_DEV(portid) {
3000                         if ((port_mask & (1 << portid)) == 0)
3001                                 continue;
3002                         memset(&link, 0, sizeof(link));
3003                         ret = rte_eth_link_get_nowait(portid, &link);
3004                         if (ret < 0) {
3005                                 all_ports_up = 0;
3006                                 if (print_flag == 1)
3007                                         printf("Port %u link get failed: %s\n",
3008                                                 portid, rte_strerror(-ret));
3009                                 continue;
3010                         }
3011                         /* print link status if flag set */
3012                         if (print_flag == 1) {
3013                                 if (link.link_status)
3014                                         printf(
3015                                         "Port%d Link Up. speed %u Mbps- %s\n",
3016                                         portid, link.link_speed,
3017                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3018                                         ("full-duplex") : ("half-duplex"));
3019                                 else
3020                                         printf("Port %d Link Down\n", portid);
3021                                 continue;
3022                         }
3023                         /* clear all_ports_up flag if any link down */
3024                         if (link.link_status == ETH_LINK_DOWN) {
3025                                 all_ports_up = 0;
3026                                 break;
3027                         }
3028                 }
3029                 /* after finally printing all link status, get out */
3030                 if (print_flag == 1)
3031                         break;
3032
3033                 if (all_ports_up == 0) {
3034                         fflush(stdout);
3035                         rte_delay_ms(CHECK_INTERVAL);
3036                 }
3037
3038                 /* set the print_flag if all ports up or timeout */
3039                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3040                         print_flag = 1;
3041                 }
3042
3043                 if (lsc_interrupt)
3044                         break;
3045         }
3046 }
3047
3048 /*
3049  * This callback is for remove a port for a device. It has limitation because
3050  * it is not for multiple port removal for a device.
3051  * TODO: the device detach invoke will plan to be removed from user side to
3052  * eal. And convert all PMDs to free port resources on ether device closing.
3053  */
3054 static void
3055 rmv_port_callback(void *arg)
3056 {
3057         int need_to_start = 0;
3058         int org_no_link_check = no_link_check;
3059         portid_t port_id = (intptr_t)arg;
3060         struct rte_device *dev;
3061
3062         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3063
3064         if (!test_done && port_is_forwarding(port_id)) {
3065                 need_to_start = 1;
3066                 stop_packet_forwarding();
3067         }
3068         no_link_check = 1;
3069         stop_port(port_id);
3070         no_link_check = org_no_link_check;
3071
3072         /* Save rte_device pointer before closing ethdev port */
3073         dev = rte_eth_devices[port_id].device;
3074         close_port(port_id);
3075         detach_device(dev); /* might be already removed or have more ports */
3076
3077         if (need_to_start)
3078                 start_packet_forwarding(0);
3079 }
3080
3081 /* This function is used by the interrupt thread */
3082 static int
3083 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3084                   void *ret_param)
3085 {
3086         RTE_SET_USED(param);
3087         RTE_SET_USED(ret_param);
3088
3089         if (type >= RTE_ETH_EVENT_MAX) {
3090                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3091                         port_id, __func__, type);
3092                 fflush(stderr);
3093         } else if (event_print_mask & (UINT32_C(1) << type)) {
3094                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3095                         eth_event_desc[type]);
3096                 fflush(stdout);
3097         }
3098
3099         switch (type) {
3100         case RTE_ETH_EVENT_NEW:
3101                 ports[port_id].need_setup = 1;
3102                 ports[port_id].port_status = RTE_PORT_HANDLING;
3103                 break;
3104         case RTE_ETH_EVENT_INTR_RMV:
3105                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3106                         break;
3107                 if (rte_eal_alarm_set(100000,
3108                                 rmv_port_callback, (void *)(intptr_t)port_id))
3109                         fprintf(stderr, "Could not set up deferred device removal\n");
3110                 break;
3111         default:
3112                 break;
3113         }
3114         return 0;
3115 }
3116
3117 static int
3118 register_eth_event_callback(void)
3119 {
3120         int ret;
3121         enum rte_eth_event_type event;
3122
3123         for (event = RTE_ETH_EVENT_UNKNOWN;
3124                         event < RTE_ETH_EVENT_MAX; event++) {
3125                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3126                                 event,
3127                                 eth_event_callback,
3128                                 NULL);
3129                 if (ret != 0) {
3130                         TESTPMD_LOG(ERR, "Failed to register callback for "
3131                                         "%s event\n", eth_event_desc[event]);
3132                         return -1;
3133                 }
3134         }
3135
3136         return 0;
3137 }
3138
3139 /* This function is used by the interrupt thread */
3140 static void
3141 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3142                              __rte_unused void *arg)
3143 {
3144         uint16_t port_id;
3145         int ret;
3146
3147         if (type >= RTE_DEV_EVENT_MAX) {
3148                 fprintf(stderr, "%s called upon invalid event %d\n",
3149                         __func__, type);
3150                 fflush(stderr);
3151         }
3152
3153         switch (type) {
3154         case RTE_DEV_EVENT_REMOVE:
3155                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3156                         device_name);
3157                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3158                 if (ret) {
3159                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3160                                 device_name);
3161                         return;
3162                 }
3163                 /*
3164                  * Because the user's callback is invoked in eal interrupt
3165                  * callback, the interrupt callback need to be finished before
3166                  * it can be unregistered when detaching device. So finish
3167                  * callback soon and use a deferred removal to detach device
3168                  * is need. It is a workaround, once the device detaching be
3169                  * moved into the eal in the future, the deferred removal could
3170                  * be deleted.
3171                  */
3172                 if (rte_eal_alarm_set(100000,
3173                                 rmv_port_callback, (void *)(intptr_t)port_id))
3174                         RTE_LOG(ERR, EAL,
3175                                 "Could not set up deferred device removal\n");
3176                 break;
3177         case RTE_DEV_EVENT_ADD:
3178                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3179                         device_name);
3180                 /* TODO: After finish kernel driver binding,
3181                  * begin to attach port.
3182                  */
3183                 break;
3184         default:
3185                 break;
3186         }
3187 }
3188
3189 static int
3190 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3191 {
3192         uint16_t i;
3193         int diag;
3194         uint8_t mapping_found = 0;
3195
3196         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3197                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3198                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3199                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3200                                         tx_queue_stats_mappings[i].queue_id,
3201                                         tx_queue_stats_mappings[i].stats_counter_id);
3202                         if (diag != 0)
3203                                 return diag;
3204                         mapping_found = 1;
3205                 }
3206         }
3207         if (mapping_found)
3208                 port->tx_queue_stats_mapping_enabled = 1;
3209         return 0;
3210 }
3211
3212 static int
3213 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3214 {
3215         uint16_t i;
3216         int diag;
3217         uint8_t mapping_found = 0;
3218
3219         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3220                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3221                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3222                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3223                                         rx_queue_stats_mappings[i].queue_id,
3224                                         rx_queue_stats_mappings[i].stats_counter_id);
3225                         if (diag != 0)
3226                                 return diag;
3227                         mapping_found = 1;
3228                 }
3229         }
3230         if (mapping_found)
3231                 port->rx_queue_stats_mapping_enabled = 1;
3232         return 0;
3233 }
3234
3235 static void
3236 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3237 {
3238         int diag = 0;
3239
3240         diag = set_tx_queue_stats_mapping_registers(pi, port);
3241         if (diag != 0) {
3242                 if (diag == -ENOTSUP) {
3243                         port->tx_queue_stats_mapping_enabled = 0;
3244                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3245                 }
3246                 else
3247                         rte_exit(EXIT_FAILURE,
3248                                         "set_tx_queue_stats_mapping_registers "
3249                                         "failed for port id=%d diag=%d\n",
3250                                         pi, diag);
3251         }
3252
3253         diag = set_rx_queue_stats_mapping_registers(pi, port);
3254         if (diag != 0) {
3255                 if (diag == -ENOTSUP) {
3256                         port->rx_queue_stats_mapping_enabled = 0;
3257                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3258                 }
3259                 else
3260                         rte_exit(EXIT_FAILURE,
3261                                         "set_rx_queue_stats_mapping_registers "
3262                                         "failed for port id=%d diag=%d\n",
3263                                         pi, diag);
3264         }
3265 }
3266
3267 static void
3268 rxtx_port_config(struct rte_port *port)
3269 {
3270         uint16_t qid;
3271         uint64_t offloads;
3272
3273         for (qid = 0; qid < nb_rxq; qid++) {
3274                 offloads = port->rx_conf[qid].offloads;
3275                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3276                 if (offloads != 0)
3277                         port->rx_conf[qid].offloads = offloads;
3278
3279                 /* Check if any Rx parameters have been passed */
3280                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3281                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3282
3283                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3284                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3285
3286                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3287                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3288
3289                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3290                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3291
3292                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3293                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3294
3295                 port->nb_rx_desc[qid] = nb_rxd;
3296         }
3297
3298         for (qid = 0; qid < nb_txq; qid++) {
3299                 offloads = port->tx_conf[qid].offloads;
3300                 port->tx_conf[qid] = port->dev_info.default_txconf;
3301                 if (offloads != 0)
3302                         port->tx_conf[qid].offloads = offloads;
3303
3304                 /* Check if any Tx parameters have been passed */
3305                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3306                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3307
3308                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3309                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3310
3311                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3312                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3313
3314                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3315                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3316
3317                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3318                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3319
3320                 port->nb_tx_desc[qid] = nb_txd;
3321         }
3322 }
3323
3324 void
3325 init_port_config(void)
3326 {
3327         portid_t pid;
3328         struct rte_port *port;
3329         int ret;
3330
3331         RTE_ETH_FOREACH_DEV(pid) {
3332                 port = &ports[pid];
3333                 port->dev_conf.fdir_conf = fdir_conf;
3334
3335                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3336                 if (ret != 0)
3337                         return;
3338
3339                 if (nb_rxq > 1) {
3340                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3341                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3342                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3343                 } else {
3344                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3345                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3346                 }
3347
3348                 if (port->dcb_flag == 0) {
3349                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3350                                 port->dev_conf.rxmode.mq_mode =
3351                                         (enum rte_eth_rx_mq_mode)
3352                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3353                         else
3354                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3355                 }
3356
3357                 rxtx_port_config(port);
3358
3359                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3360                 if (ret != 0)
3361                         return;
3362
3363                 map_port_queue_stats_mapping_registers(pid, port);
3364 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3365                 rte_pmd_ixgbe_bypass_init(pid);
3366 #endif
3367
3368                 if (lsc_interrupt &&
3369                     (rte_eth_devices[pid].data->dev_flags &
3370                      RTE_ETH_DEV_INTR_LSC))
3371                         port->dev_conf.intr_conf.lsc = 1;
3372                 if (rmv_interrupt &&
3373                     (rte_eth_devices[pid].data->dev_flags &
3374                      RTE_ETH_DEV_INTR_RMV))
3375                         port->dev_conf.intr_conf.rmv = 1;
3376         }
3377 }
3378
3379 void set_port_slave_flag(portid_t slave_pid)
3380 {
3381         struct rte_port *port;
3382
3383         port = &ports[slave_pid];
3384         port->slave_flag = 1;
3385 }
3386
3387 void clear_port_slave_flag(portid_t slave_pid)
3388 {
3389         struct rte_port *port;
3390
3391         port = &ports[slave_pid];
3392         port->slave_flag = 0;
3393 }
3394
3395 uint8_t port_is_bonding_slave(portid_t slave_pid)
3396 {
3397         struct rte_port *port;
3398
3399         port = &ports[slave_pid];
3400         if ((rte_eth_devices[slave_pid].data->dev_flags &
3401             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3402                 return 1;
3403         return 0;
3404 }
3405
3406 const uint16_t vlan_tags[] = {
3407                 0,  1,  2,  3,  4,  5,  6,  7,
3408                 8,  9, 10, 11,  12, 13, 14, 15,
3409                 16, 17, 18, 19, 20, 21, 22, 23,
3410                 24, 25, 26, 27, 28, 29, 30, 31
3411 };
3412
3413 static  int
3414 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3415                  enum dcb_mode_enable dcb_mode,
3416                  enum rte_eth_nb_tcs num_tcs,
3417                  uint8_t pfc_en)
3418 {
3419         uint8_t i;
3420         int32_t rc;
3421         struct rte_eth_rss_conf rss_conf;
3422
3423         /*
3424          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3425          * given above, and the number of traffic classes available for use.
3426          */
3427         if (dcb_mode == DCB_VT_ENABLED) {
3428                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3429                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3430                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3431                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3432
3433                 /* VMDQ+DCB RX and TX configurations */
3434                 vmdq_rx_conf->enable_default_pool = 0;
3435                 vmdq_rx_conf->default_pool = 0;
3436                 vmdq_rx_conf->nb_queue_pools =
3437                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3438                 vmdq_tx_conf->nb_queue_pools =
3439                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3440
3441                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3442                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3443                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3444                         vmdq_rx_conf->pool_map[i].pools =
3445                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3446                 }
3447                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3448                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3449                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3450                 }
3451
3452                 /* set DCB mode of RX and TX of multiple queues */
3453                 eth_conf->rxmode.mq_mode =
3454                                 (enum rte_eth_rx_mq_mode)
3455                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3456                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3457         } else {
3458                 struct rte_eth_dcb_rx_conf *rx_conf =
3459                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3460                 struct rte_eth_dcb_tx_conf *tx_conf =
3461                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3462
3463                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3464                 if (rc != 0)
3465                         return rc;
3466
3467                 rx_conf->nb_tcs = num_tcs;
3468                 tx_conf->nb_tcs = num_tcs;
3469
3470                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3471                         rx_conf->dcb_tc[i] = i % num_tcs;
3472                         tx_conf->dcb_tc[i] = i % num_tcs;
3473                 }
3474
3475                 eth_conf->rxmode.mq_mode =
3476                                 (enum rte_eth_rx_mq_mode)
3477                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3478                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3479                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3480         }
3481
3482         if (pfc_en)
3483                 eth_conf->dcb_capability_en =
3484                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3485         else
3486                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3487
3488         return 0;
3489 }
3490
3491 int
3492 init_port_dcb_config(portid_t pid,
3493                      enum dcb_mode_enable dcb_mode,
3494                      enum rte_eth_nb_tcs num_tcs,
3495                      uint8_t pfc_en)
3496 {
3497         struct rte_eth_conf port_conf;
3498         struct rte_port *rte_port;
3499         int retval;
3500         uint16_t i;
3501
3502         rte_port = &ports[pid];
3503
3504         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3505         /* Enter DCB configuration status */
3506         dcb_config = 1;
3507
3508         port_conf.rxmode = rte_port->dev_conf.rxmode;
3509         port_conf.txmode = rte_port->dev_conf.txmode;
3510
3511         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3512         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3513         if (retval < 0)
3514                 return retval;
3515         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3516
3517         /* re-configure the device . */
3518         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3519         if (retval < 0)
3520                 return retval;
3521
3522         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3523         if (retval != 0)
3524                 return retval;
3525
3526         /* If dev_info.vmdq_pool_base is greater than 0,
3527          * the queue id of vmdq pools is started after pf queues.
3528          */
3529         if (dcb_mode == DCB_VT_ENABLED &&
3530             rte_port->dev_info.vmdq_pool_base > 0) {
3531                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3532                         " for port %d.", pid);
3533                 return -1;
3534         }
3535
3536         /* Assume the ports in testpmd have the same dcb capability
3537          * and has the same number of rxq and txq in dcb mode
3538          */
3539         if (dcb_mode == DCB_VT_ENABLED) {
3540                 if (rte_port->dev_info.max_vfs > 0) {
3541                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3542                         nb_txq = rte_port->dev_info.nb_tx_queues;
3543                 } else {
3544                         nb_rxq = rte_port->dev_info.max_rx_queues;
3545                         nb_txq = rte_port->dev_info.max_tx_queues;
3546                 }
3547         } else {
3548                 /*if vt is disabled, use all pf queues */
3549                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3550                         nb_rxq = rte_port->dev_info.max_rx_queues;
3551                         nb_txq = rte_port->dev_info.max_tx_queues;
3552                 } else {
3553                         nb_rxq = (queueid_t)num_tcs;
3554                         nb_txq = (queueid_t)num_tcs;
3555
3556                 }
3557         }
3558         rx_free_thresh = 64;
3559
3560         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3561
3562         rxtx_port_config(rte_port);
3563         /* VLAN filter */
3564         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3565         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3566                 rx_vft_set(pid, vlan_tags[i], 1);
3567
3568         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3569         if (retval != 0)
3570                 return retval;
3571
3572         map_port_queue_stats_mapping_registers(pid, rte_port);
3573
3574         rte_port->dcb_flag = 1;
3575
3576         return 0;
3577 }
3578
3579 static void
3580 init_port(void)
3581 {
3582         /* Configuration of Ethernet ports. */
3583         ports = rte_zmalloc("testpmd: ports",
3584                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3585                             RTE_CACHE_LINE_SIZE);
3586         if (ports == NULL) {
3587                 rte_exit(EXIT_FAILURE,
3588                                 "rte_zmalloc(%d struct rte_port) failed\n",
3589                                 RTE_MAX_ETHPORTS);
3590         }
3591
3592         /* Initialize ports NUMA structures */
3593         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3594         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3595         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3596 }
3597
3598 static void
3599 force_quit(void)
3600 {
3601         pmd_test_exit();
3602         prompt_exit();
3603 }
3604
3605 static void
3606 print_stats(void)
3607 {
3608         uint8_t i;
3609         const char clr[] = { 27, '[', '2', 'J', '\0' };
3610         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3611
3612         /* Clear screen and move to top left */
3613         printf("%s%s", clr, top_left);
3614
3615         printf("\nPort statistics ====================================");
3616         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3617                 nic_stats_display(fwd_ports_ids[i]);
3618
3619         fflush(stdout);
3620 }
3621
3622 static void
3623 signal_handler(int signum)
3624 {
3625         if (signum == SIGINT || signum == SIGTERM) {
3626                 printf("\nSignal %d received, preparing to exit...\n",
3627                                 signum);
3628 #ifdef RTE_LIBRTE_PDUMP
3629                 /* uninitialize packet capture framework */
3630                 rte_pdump_uninit();
3631 #endif
3632 #ifdef RTE_LIBRTE_LATENCY_STATS
3633                 if (latencystats_enabled != 0)
3634                         rte_latencystats_uninit();
3635 #endif
3636                 force_quit();
3637                 /* Set flag to indicate the force termination. */
3638                 f_quit = 1;
3639                 /* exit with the expected status */
3640                 signal(signum, SIG_DFL);
3641                 kill(getpid(), signum);
3642         }
3643 }
3644
3645 int
3646 main(int argc, char** argv)
3647 {
3648         int diag;
3649         portid_t port_id;
3650         uint16_t count;
3651         int ret;
3652
3653         signal(SIGINT, signal_handler);
3654         signal(SIGTERM, signal_handler);
3655
3656         testpmd_logtype = rte_log_register("testpmd");
3657         if (testpmd_logtype < 0)
3658                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3659         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3660
3661         diag = rte_eal_init(argc, argv);
3662         if (diag < 0)
3663                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3664                          rte_strerror(rte_errno));
3665
3666         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3667                 rte_exit(EXIT_FAILURE,
3668                          "Secondary process type not supported.\n");
3669
3670         ret = register_eth_event_callback();
3671         if (ret != 0)
3672                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3673
3674 #ifdef RTE_LIBRTE_PDUMP
3675         /* initialize packet capture framework */
3676         rte_pdump_init();
3677 #endif
3678
3679         count = 0;
3680         RTE_ETH_FOREACH_DEV(port_id) {
3681                 ports_ids[count] = port_id;
3682                 count++;
3683         }
3684         nb_ports = (portid_t) count;
3685         if (nb_ports == 0)
3686                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3687
3688         /* allocate port structures, and init them */
3689         init_port();
3690
3691         set_def_fwd_config();
3692         if (nb_lcores == 0)
3693                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3694                          "Check the core mask argument\n");
3695
3696         /* Bitrate/latency stats disabled by default */
3697 #ifdef RTE_LIBRTE_BITRATE
3698         bitrate_enabled = 0;
3699 #endif
3700 #ifdef RTE_LIBRTE_LATENCY_STATS
3701         latencystats_enabled = 0;
3702 #endif
3703
3704         /* on FreeBSD, mlockall() is disabled by default */
3705 #ifdef RTE_EXEC_ENV_FREEBSD
3706         do_mlockall = 0;
3707 #else
3708         do_mlockall = 1;
3709 #endif
3710
3711         argc -= diag;
3712         argv += diag;
3713         if (argc > 1)
3714                 launch_args_parse(argc, argv);
3715
3716         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3717                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3718                         strerror(errno));
3719         }
3720
3721         if (tx_first && interactive)
3722                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3723                                 "interactive mode.\n");
3724
3725         if (tx_first && lsc_interrupt) {
3726                 printf("Warning: lsc_interrupt needs to be off when "
3727                                 " using tx_first. Disabling.\n");
3728                 lsc_interrupt = 0;
3729         }
3730
3731         if (!nb_rxq && !nb_txq)
3732                 printf("Warning: Either rx or tx queues should be non-zero\n");
3733
3734         if (nb_rxq > 1 && nb_rxq > nb_txq)
3735                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3736                        "but nb_txq=%d will prevent to fully test it.\n",
3737                        nb_rxq, nb_txq);
3738
3739         init_config();
3740
3741         if (hot_plug) {
3742                 ret = rte_dev_hotplug_handle_enable();
3743                 if (ret) {
3744                         RTE_LOG(ERR, EAL,
3745                                 "fail to enable hotplug handling.");
3746                         return -1;
3747                 }
3748
3749                 ret = rte_dev_event_monitor_start();
3750                 if (ret) {
3751                         RTE_LOG(ERR, EAL,
3752                                 "fail to start device event monitoring.");
3753                         return -1;
3754                 }
3755
3756                 ret = rte_dev_event_callback_register(NULL,
3757                         dev_event_callback, NULL);
3758                 if (ret) {
3759                         RTE_LOG(ERR, EAL,
3760                                 "fail  to register device event callback\n");
3761                         return -1;
3762                 }
3763         }
3764
3765         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3766                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3767
3768         /* set all ports to promiscuous mode by default */
3769         RTE_ETH_FOREACH_DEV(port_id) {
3770                 ret = rte_eth_promiscuous_enable(port_id);
3771                 if (ret != 0)
3772                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3773                                 port_id, rte_strerror(-ret));
3774         }
3775
3776         /* Init metrics library */
3777         rte_metrics_init(rte_socket_id());
3778
3779 #ifdef RTE_LIBRTE_LATENCY_STATS
3780         if (latencystats_enabled != 0) {
3781                 int ret = rte_latencystats_init(1, NULL);
3782                 if (ret)
3783                         printf("Warning: latencystats init()"
3784                                 " returned error %d\n", ret);
3785                 printf("Latencystats running on lcore %d\n",
3786                         latencystats_lcore_id);
3787         }
3788 #endif
3789
3790         /* Setup bitrate stats */
3791 #ifdef RTE_LIBRTE_BITRATE
3792         if (bitrate_enabled != 0) {
3793                 bitrate_data = rte_stats_bitrate_create();
3794                 if (bitrate_data == NULL)
3795                         rte_exit(EXIT_FAILURE,
3796                                 "Could not allocate bitrate data.\n");
3797                 rte_stats_bitrate_reg(bitrate_data);
3798         }
3799 #endif
3800
3801 #ifdef RTE_LIBRTE_CMDLINE
3802         if (strlen(cmdline_filename) != 0)
3803                 cmdline_read_from_file(cmdline_filename);
3804
3805         if (interactive == 1) {
3806                 if (auto_start) {
3807                         printf("Start automatic packet forwarding\n");
3808                         start_packet_forwarding(0);
3809                 }
3810                 prompt();
3811                 pmd_test_exit();
3812         } else
3813 #endif
3814         {
3815                 char c;
3816                 int rc;
3817
3818                 f_quit = 0;
3819
3820                 printf("No commandline core given, start packet forwarding\n");
3821                 start_packet_forwarding(tx_first);
3822                 if (stats_period != 0) {
3823                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3824                         uint64_t timer_period;
3825
3826                         /* Convert to number of cycles */
3827                         timer_period = stats_period * rte_get_timer_hz();
3828
3829                         while (f_quit == 0) {
3830                                 cur_time = rte_get_timer_cycles();
3831                                 diff_time += cur_time - prev_time;
3832
3833                                 if (diff_time >= timer_period) {
3834                                         print_stats();
3835                                         /* Reset the timer */
3836                                         diff_time = 0;
3837                                 }
3838                                 /* Sleep to avoid unnecessary checks */
3839                                 prev_time = cur_time;
3840                                 sleep(1);
3841                         }
3842                 }
3843
3844                 printf("Press enter to exit\n");
3845                 rc = read(0, &c, 1);
3846                 pmd_test_exit();
3847                 if (rc < 0)
3848                         return 1;
3849         }
3850
3851         ret = rte_eal_cleanup();
3852         if (ret != 0)
3853                 rte_exit(EXIT_FAILURE,
3854                          "EAL cleanup failed: %s\n", strerror(-ret));
3855
3856         return EXIT_SUCCESS;
3857 }