app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #if defined RTE_LIBRTE_PMD_SOFTNIC
 183         &softnic_fwd_engine,
 184 #endif
 185 #ifdef RTE_LIBRTE_IEEE1588
 186         &ieee1588_fwd_engine,
 187 #endif
 188         NULL,
 189 };
 190
 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 192 uint16_t mempool_flags;
 193
 194 struct fwd_config cur_fwd_config;
 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 196 uint32_t retry_enabled;
 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 199
 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 201 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 202                                       * specified on command-line. */
 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 204
 205 /*
 206  * In container, it cannot terminate the process which running with 'stats-period'
 207  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 208  */
 209 uint8_t f_quit;
 210
 211 /*
 212  * Configuration of packet segments used by the "txonly" processing engine.
 213  */
 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 216         TXONLY_DEF_PACKET_LEN,
 217 };
 218 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 219
 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 221 /**< Split policy for packets to TX. */
 222
 223 uint8_t txonly_multi_flow;
 224 /**< Whether multiple flows are generated in TXONLY mode. */
 225
 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 228
 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 230 uint8_t dcb_config = 0;
 231
 232 /* Whether the dcb is in testing status */
 233 uint8_t dcb_test = 0;
 234
 235 /*
 236  * Configurable number of RX/TX queues.
 237  */
 238 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 239 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 240 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 241
 242 /*
 243  * Configurable number of RX/TX ring descriptors.
 244  * Defaults are supplied by drivers via ethdev.
 245  */
 246 #define RTE_TEST_RX_DESC_DEFAULT 0
 247 #define RTE_TEST_TX_DESC_DEFAULT 0
 248 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 249 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 250
 251 #define RTE_PMD_PARAM_UNSET -1
 252 /*
 253  * Configurable values of RX and TX ring threshold registers.
 254  */
 255
 256 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 257 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 258 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 259
 260 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 261 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 262 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 263
 264 /*
 265  * Configurable value of RX free threshold.
 266  */
 267 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 268
 269 /*
 270  * Configurable value of RX drop enable.
 271  */
 272 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 273
 274 /*
 275  * Configurable value of TX free threshold.
 276  */
 277 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 278
 279 /*
 280  * Configurable value of TX RS bit threshold.
 281  */
 282 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 283
 284 /*
 285  * Configurable value of buffered packets before sending.
 286  */
 287 uint16_t noisy_tx_sw_bufsz;
 288
 289 /*
 290  * Configurable value of packet buffer timeout.
 291  */
 292 uint16_t noisy_tx_sw_buf_flush_time;
 293
 294 /*
 295  * Configurable value for size of VNF internal memory area
 296  * used for simulating noisy neighbour behaviour
 297  */
 298 uint64_t noisy_lkup_mem_sz;
 299
 300 /*
 301  * Configurable value of number of random writes done in
 302  * VNF simulation memory area.
 303  */
 304 uint64_t noisy_lkup_num_writes;
 305
 306 /*
 307  * Configurable value of number of random reads done in
 308  * VNF simulation memory area.
 309  */
 310 uint64_t noisy_lkup_num_reads;
 311
 312 /*
 313  * Configurable value of number of random reads/writes done in
 314  * VNF simulation memory area.
 315  */
 316 uint64_t noisy_lkup_num_reads_writes;
 317
 318 /*
 319  * Receive Side Scaling (RSS) configuration.
 320  */
 321 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 322
 323 /*
 324  * Port topology configuration
 325  */
 326 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 327
 328 /*
 329  * Avoids to flush all the RX streams before starts forwarding.
 330  */
 331 uint8_t no_flush_rx = 0; /* flush by default */
 332
 333 /*
 334  * Flow API isolated mode.
 335  */
 336 uint8_t flow_isolate_all;
 337
 338 /*
 339  * Avoids to check link status when starting/stopping a port.
 340  */
 341 uint8_t no_link_check = 0; /* check by default */
 342
 343 /*
 344  * Don't automatically start all ports in interactive mode.
 345  */
 346 uint8_t no_device_start = 0;
 347
 348 /*
 349  * Enable link status change notification
 350  */
 351 uint8_t lsc_interrupt = 1; /* enabled by default */
 352
 353 /*
 354  * Enable device removal notification.
 355  */
 356 uint8_t rmv_interrupt = 1; /* enabled by default */
 357
 358 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 359
 360 /* After attach, port setup is called on event or by iterator */
 361 bool setup_on_probe_event = true;
 362
 363 /* Clear ptypes on port initialization. */
 364 uint8_t clear_ptypes = true;
 365
 366 /* Pretty printing of ethdev events */
 367 static const char * const eth_event_desc[] = {
 368         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 369         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 370         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 371         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 372         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 373         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 374         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 375         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 376         [RTE_ETH_EVENT_NEW] = "device probed",
 377         [RTE_ETH_EVENT_DESTROY] = "device released",
 378         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 379         [RTE_ETH_EVENT_MAX] = NULL,
 380 };
 381
 382 /*
 383  * Display or mask ether events
 384  * Default to all events except VF_MBOX
 385  */
 386 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 388                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 389                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 390                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 391                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 392                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 393                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 394 /*
 395  * Decide if all memory are locked for performance.
 396  */
 397 int do_mlockall = 0;
 398
 399 /*
 400  * NIC bypass mode configuration options.
 401  */
 402
 403 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 404 /* The NIC bypass watchdog timeout. */
 405 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 406 #endif
 407
 408
 409 #ifdef RTE_LIBRTE_LATENCY_STATS
 410
 411 /*
 412  * Set when latency stats is enabled in the commandline
 413  */
 414 uint8_t latencystats_enabled;
 415
 416 /*
 417  * Lcore ID to serive latency statistics.
 418  */
 419 lcoreid_t latencystats_lcore_id = -1;
 420
 421 #endif
 422
 423 /*
 424  * Ethernet device configuration.
 425  */
 426 struct rte_eth_rxmode rx_mode = {
 427         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 428                 /**< Default maximum frame length. */
 429 };
 430
 431 struct rte_eth_txmode tx_mode = {
 432         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 433 };
 434
 435 struct rte_fdir_conf fdir_conf = {
 436         .mode = RTE_FDIR_MODE_NONE,
 437         .pballoc = RTE_FDIR_PBALLOC_64K,
 438         .status = RTE_FDIR_REPORT_STATUS,
 439         .mask = {
 440                 .vlan_tci_mask = 0xFFEF,
 441                 .ipv4_mask     = {
 442                         .src_ip = 0xFFFFFFFF,
 443                         .dst_ip = 0xFFFFFFFF,
 444                 },
 445                 .ipv6_mask     = {
 446                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 447                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 448                 },
 449                 .src_port_mask = 0xFFFF,
 450                 .dst_port_mask = 0xFFFF,
 451                 .mac_addr_byte_mask = 0xFF,
 452                 .tunnel_type_mask = 1,
 453                 .tunnel_id_mask = 0xFFFFFFFF,
 454         },
 455         .drop_queue = 127,
 456 };
 457
 458 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 459
 460 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 461 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 462
 463 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 464 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 465
 466 uint16_t nb_tx_queue_stats_mappings = 0;
 467 uint16_t nb_rx_queue_stats_mappings = 0;
 468
 469 /*
 470  * Display zero values by default for xstats
 471  */
 472 uint8_t xstats_hide_zero;
 473
 474 unsigned int num_sockets = 0;
 475 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 476
 477 #ifdef RTE_LIBRTE_BITRATE
 478 /* Bitrate statistics */
 479 struct rte_stats_bitrates *bitrate_data;
 480 lcoreid_t bitrate_lcore_id;
 481 uint8_t bitrate_enabled;
 482 #endif
 483
 484 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 485 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 486
 487 /*
 488  * hexadecimal bitmask of RX mq mode can be enabled.
 489  */
 490 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 491
 492 /* Forward function declarations */
 493 static void setup_attached_port(portid_t pi);
 494 static void map_port_queue_stats_mapping_registers(portid_t pi,
 495                                                    struct rte_port *port);
 496 static void check_all_ports_link_status(uint32_t port_mask);
 497 static int eth_event_callback(portid_t port_id,
 498                               enum rte_eth_event_type type,
 499                               void *param, void *ret_param);
 500 static void dev_event_callback(const char *device_name,
 501                                 enum rte_dev_event_type type,
 502                                 void *param);
 503
 504 /*
 505  * Check if all the ports are started.
 506  * If yes, return positive value. If not, return zero.
 507  */
 508 static int all_ports_started(void);
 509
 510 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 511 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 512
 513 /* Holds the registered mbuf dynamic flags names. */
 514 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 515
 516 /*
 517  * Helper function to check if socket is already discovered.
 518  * If yes, return positive value. If not, return zero.
 519  */
 520 int
 521 new_socket_id(unsigned int socket_id)
 522 {
 523         unsigned int i;
 524
 525         for (i = 0; i < num_sockets; i++) {
 526                 if (socket_ids[i] == socket_id)
 527                         return 0;
 528         }
 529         return 1;
 530 }
 531
 532 /*
 533  * Setup default configuration.
 534  */
 535 static void
 536 set_default_fwd_lcores_config(void)
 537 {
 538         unsigned int i;
 539         unsigned int nb_lc;
 540         unsigned int sock_num;
 541
 542         nb_lc = 0;
 543         for (i = 0; i < RTE_MAX_LCORE; i++) {
 544                 if (!rte_lcore_is_enabled(i))
 545                         continue;
 546                 sock_num = rte_lcore_to_socket_id(i);
 547                 if (new_socket_id(sock_num)) {
 548                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 549                                 rte_exit(EXIT_FAILURE,
 550                                          "Total sockets greater than %u\n",
 551                                          RTE_MAX_NUMA_NODES);
 552                         }
 553                         socket_ids[num_sockets++] = sock_num;
 554                 }
 555                 if (i == rte_get_master_lcore())
 556                         continue;
 557                 fwd_lcores_cpuids[nb_lc++] = i;
 558         }
 559         nb_lcores = (lcoreid_t) nb_lc;
 560         nb_cfg_lcores = nb_lcores;
 561         nb_fwd_lcores = 1;
 562 }
 563
 564 static void
 565 set_def_peer_eth_addrs(void)
 566 {
 567         portid_t i;
 568
 569         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 570                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 571                 peer_eth_addrs[i].addr_bytes[5] = i;
 572         }
 573 }
 574
 575 static void
 576 set_default_fwd_ports_config(void)
 577 {
 578         portid_t pt_id;
 579         int i = 0;
 580
 581         RTE_ETH_FOREACH_DEV(pt_id) {
 582                 fwd_ports_ids[i++] = pt_id;
 583
 584                 /* Update sockets info according to the attached device */
 585                 int socket_id = rte_eth_dev_socket_id(pt_id);
 586                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 587                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 588                                 rte_exit(EXIT_FAILURE,
 589                                          "Total sockets greater than %u\n",
 590                                          RTE_MAX_NUMA_NODES);
 591                         }
 592                         socket_ids[num_sockets++] = socket_id;
 593                 }
 594         }
 595
 596         nb_cfg_ports = nb_ports;
 597         nb_fwd_ports = nb_ports;
 598 }
 599
 600 void
 601 set_def_fwd_config(void)
 602 {
 603         set_default_fwd_lcores_config();
 604         set_def_peer_eth_addrs();
 605         set_default_fwd_ports_config();
 606 }
 607
 608 /* extremely pessimistic estimation of memory required to create a mempool */
 609 static int
 610 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 611 {
 612         unsigned int n_pages, mbuf_per_pg, leftover;
 613         uint64_t total_mem, mbuf_mem, obj_sz;
 614
 615         /* there is no good way to predict how much space the mempool will
 616          * occupy because it will allocate chunks on the fly, and some of those
 617          * will come from default DPDK memory while some will come from our
 618          * external memory, so just assume 128MB will be enough for everyone.
 619          */
 620         uint64_t hdr_mem = 128 << 20;
 621
 622         /* account for possible non-contiguousness */
 623         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 624         if (obj_sz > pgsz) {
 625                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 626                 return -1;
 627         }
 628
 629         mbuf_per_pg = pgsz / obj_sz;
 630         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 631         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 632
 633         mbuf_mem = n_pages * pgsz;
 634
 635         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 636
 637         if (total_mem > SIZE_MAX) {
 638                 TESTPMD_LOG(ERR, "Memory size too big\n");
 639                 return -1;
 640         }
 641         *out = (size_t)total_mem;
 642
 643         return 0;
 644 }
 645
 646 static int
 647 pagesz_flags(uint64_t page_sz)
 648 {
 649         /* as per mmap() manpage, all page sizes are log2 of page size
 650          * shifted by MAP_HUGE_SHIFT
 651          */
 652         int log2 = rte_log2_u64(page_sz);
 653
 654         return (log2 << HUGE_SHIFT);
 655 }
 656
 657 static void *
 658 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 659 {
 660         void *addr;
 661         int flags;
 662
 663         /* allocate anonymous hugepages */
 664         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 665         if (huge)
 666                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 667
 668         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 669         if (addr == MAP_FAILED)
 670                 return NULL;
 671
 672         return addr;
 673 }
 674
 675 struct extmem_param {
 676         void *addr;
 677         size_t len;
 678         size_t pgsz;
 679         rte_iova_t *iova_table;
 680         unsigned int iova_table_len;
 681 };
 682
 683 static int
 684 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 685                 bool huge)
 686 {
 687         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 688                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 689         unsigned int cur_page, n_pages, pgsz_idx;
 690         size_t mem_sz, cur_pgsz;
 691         rte_iova_t *iovas = NULL;
 692         void *addr;
 693         int ret;
 694
 695         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 696                 /* skip anything that is too big */
 697                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 698                         continue;
 699
 700                 cur_pgsz = pgsizes[pgsz_idx];
 701
 702                 /* if we were told not to allocate hugepages, override */
 703                 if (!huge)
 704                         cur_pgsz = sysconf(_SC_PAGESIZE);
 705
 706                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 707                 if (ret < 0) {
 708                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 709                         return -1;
 710                 }
 711
 712                 /* allocate our memory */
 713                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 714
 715                 /* if we couldn't allocate memory with a specified page size,
 716                  * that doesn't mean we can't do it with other page sizes, so
 717                  * try another one.
 718                  */
 719                 if (addr == NULL)
 720                         continue;
 721
 722                 /* store IOVA addresses for every page in this memory area */
 723                 n_pages = mem_sz / cur_pgsz;
 724
 725                 iovas = malloc(sizeof(*iovas) * n_pages);
 726
 727                 if (iovas == NULL) {
 728                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 729                         goto fail;
 730                 }
 731                 /* lock memory if it's not huge pages */
 732                 if (!huge)
 733                         mlock(addr, mem_sz);
 734
 735                 /* populate IOVA addresses */
 736                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 737                         rte_iova_t iova;
 738                         size_t offset;
 739                         void *cur;
 740
 741                         offset = cur_pgsz * cur_page;
 742                         cur = RTE_PTR_ADD(addr, offset);
 743
 744                         /* touch the page before getting its IOVA */
 745                         *(volatile char *)cur = 0;
 746
 747                         iova = rte_mem_virt2iova(cur);
 748
 749                         iovas[cur_page] = iova;
 750                 }
 751
 752                 break;
 753         }
 754         /* if we couldn't allocate anything */
 755         if (iovas == NULL)
 756                 return -1;
 757
 758         param->addr = addr;
 759         param->len = mem_sz;
 760         param->pgsz = cur_pgsz;
 761         param->iova_table = iovas;
 762         param->iova_table_len = n_pages;
 763
 764         return 0;
 765 fail:
 766         if (iovas)
 767                 free(iovas);
 768         if (addr)
 769                 munmap(addr, mem_sz);
 770
 771         return -1;
 772 }
 773
 774 static int
 775 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 776 {
 777         struct extmem_param param;
 778         int socket_id, ret;
 779
 780         memset(&param, 0, sizeof(param));
 781
 782         /* check if our heap exists */
 783         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 784         if (socket_id < 0) {
 785                 /* create our heap */
 786                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 787                 if (ret < 0) {
 788                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 789                         return -1;
 790                 }
 791         }
 792
 793         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 794         if (ret < 0) {
 795                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 796                 return -1;
 797         }
 798
 799         /* we now have a valid memory area, so add it to heap */
 800         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 801                         param.addr, param.len, param.iova_table,
 802                         param.iova_table_len, param.pgsz);
 803
 804         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 805
 806         /* not needed any more */
 807         free(param.iova_table);
 808
 809         if (ret < 0) {
 810                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 811                 munmap(param.addr, param.len);
 812                 return -1;
 813         }
 814
 815         /* success */
 816
 817         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 818                         param.len >> 20);
 819
 820         return 0;
 821 }
 822 static void
 823 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 824              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 825 {
 826         uint16_t pid = 0;
 827         int ret;
 828
 829         RTE_ETH_FOREACH_DEV(pid) {
 830                 struct rte_eth_dev *dev =
 831                         &rte_eth_devices[pid];
 832
 833                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 834                                         memhdr->len);
 835                 if (ret) {
 836                         TESTPMD_LOG(DEBUG,
 837                                     "unable to DMA unmap addr 0x%p "
 838                                     "for device %s\n",
 839                                     memhdr->addr, dev->data->name);
 840                 }
 841         }
 842         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 843         if (ret) {
 844                 TESTPMD_LOG(DEBUG,
 845                             "unable to un-register addr 0x%p\n", memhdr->addr);
 846         }
 847 }
 848
 849 static void
 850 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 851            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 852 {
 853         uint16_t pid = 0;
 854         size_t page_size = sysconf(_SC_PAGESIZE);
 855         int ret;
 856
 857         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 858                                   page_size);
 859         if (ret) {
 860                 TESTPMD_LOG(DEBUG,
 861                             "unable to register addr 0x%p\n", memhdr->addr);
 862                 return;
 863         }
 864         RTE_ETH_FOREACH_DEV(pid) {
 865                 struct rte_eth_dev *dev =
 866                         &rte_eth_devices[pid];
 867
 868                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 869                                       memhdr->len);
 870                 if (ret) {
 871                         TESTPMD_LOG(DEBUG,
 872                                     "unable to DMA map addr 0x%p "
 873                                     "for device %s\n",
 874                                     memhdr->addr, dev->data->name);
 875                 }
 876         }
 877 }
 878
 879 static unsigned int
 880 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 881             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 882 {
 883         struct rte_pktmbuf_extmem *xmem;
 884         unsigned int ext_num, zone_num, elt_num;
 885         uint16_t elt_size;
 886
 887         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 888         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 889         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 890
 891         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 892         if (xmem == NULL) {
 893                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 894                                  "external buffer descriptors\n");
 895                 *ext_mem = NULL;
 896                 return 0;
 897         }
 898         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 899                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 900                 const struct rte_memzone *mz;
 901                 char mz_name[RTE_MEMZONE_NAMESIZE];
 902                 int ret;
 903
 904                 ret = snprintf(mz_name, sizeof(mz_name),
 905                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 906                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 907                         errno = ENAMETOOLONG;
 908                         ext_num = 0;
 909                         break;
 910                 }
 911                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 912                                                  socket_id,
 913                                                  RTE_MEMZONE_IOVA_CONTIG |
 914                                                  RTE_MEMZONE_1GB |
 915                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 916                                                  EXTBUF_ZONE_SIZE);
 917                 if (mz == NULL) {
 918                         /*
 919                          * The caller exits on external buffer creation
 920                          * error, so there is no need to free memzones.
 921                          */
 922                         errno = ENOMEM;
 923                         ext_num = 0;
 924                         break;
 925                 }
 926                 xseg->buf_ptr = mz->addr;
 927                 xseg->buf_iova = mz->iova;
 928                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 929                 xseg->elt_size = elt_size;
 930         }
 931         if (ext_num == 0 && xmem != NULL) {
 932                 free(xmem);
 933                 xmem = NULL;
 934         }
 935         *ext_mem = xmem;
 936         return ext_num;
 937 }
 938
 939 /*
 940  * Configuration initialisation done once at init time.
 941  */
 942 static struct rte_mempool *
 943 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 944                  unsigned int socket_id)
 945 {
 946         char pool_name[RTE_MEMPOOL_NAMESIZE];
 947         struct rte_mempool *rte_mp = NULL;
 948         uint32_t mb_size;
 949
 950         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 951         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 952
 953         TESTPMD_LOG(INFO,
 954                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 955                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 956
 957         switch (mp_alloc_type) {
 958         case MP_ALLOC_NATIVE:
 959                 {
 960                         /* wrapper to rte_mempool_create() */
 961                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 962                                         rte_mbuf_best_mempool_ops());
 963                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 964                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 965                         break;
 966                 }
 967         case MP_ALLOC_ANON:
 968                 {
 969                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 970                                 mb_size, (unsigned int) mb_mempool_cache,
 971                                 sizeof(struct rte_pktmbuf_pool_private),
 972                                 socket_id, mempool_flags);
 973                         if (rte_mp == NULL)
 974                                 goto err;
 975
 976                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 977                                 rte_mempool_free(rte_mp);
 978                                 rte_mp = NULL;
 979                                 goto err;
 980                         }
 981                         rte_pktmbuf_pool_init(rte_mp, NULL);
 982                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 983                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 984                         break;
 985                 }
 986         case MP_ALLOC_XMEM:
 987         case MP_ALLOC_XMEM_HUGE:
 988                 {
 989                         int heap_socket;
 990                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 991
 992                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 993                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 994
 995                         heap_socket =
 996                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 997                         if (heap_socket < 0)
 998                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 999
1000                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1001                                         rte_mbuf_best_mempool_ops());
1002                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1003                                         mb_mempool_cache, 0, mbuf_seg_size,
1004                                         heap_socket);
1005                         break;
1006                 }
1007         case MP_ALLOC_XBUF:
1008                 {
1009                         struct rte_pktmbuf_extmem *ext_mem;
1010                         unsigned int ext_num;
1011
1012                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1013                                                socket_id, pool_name, &ext_mem);
1014                         if (ext_num == 0)
1015                                 rte_exit(EXIT_FAILURE,
1016                                          "Can't create pinned data buffers\n");
1017
1018                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1019                                         rte_mbuf_best_mempool_ops());
1020                         rte_mp = rte_pktmbuf_pool_create_extbuf
1021                                         (pool_name, nb_mbuf, mb_mempool_cache,
1022                                          0, mbuf_seg_size, socket_id,
1023                                          ext_mem, ext_num);
1024                         free(ext_mem);
1025                         break;
1026                 }
1027         default:
1028                 {
1029                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1030                 }
1031         }
1032
1033 err:
1034         if (rte_mp == NULL) {
1035                 rte_exit(EXIT_FAILURE,
1036                         "Creation of mbuf pool for socket %u failed: %s\n",
1037                         socket_id, rte_strerror(rte_errno));
1038         } else if (verbose_level > 0) {
1039                 rte_mempool_dump(stdout, rte_mp);
1040         }
1041         return rte_mp;
1042 }
1043
1044 /*
1045  * Check given socket id is valid or not with NUMA mode,
1046  * if valid, return 0, else return -1
1047  */
1048 static int
1049 check_socket_id(const unsigned int socket_id)
1050 {
1051         static int warning_once = 0;
1052
1053         if (new_socket_id(socket_id)) {
1054                 if (!warning_once && numa_support)
1055                         printf("Warning: NUMA should be configured manually by"
1056                                " using --port-numa-config and"
1057                                " --ring-numa-config parameters along with"
1058                                " --numa.\n");
1059                 warning_once = 1;
1060                 return -1;
1061         }
1062         return 0;
1063 }
1064
1065 /*
1066  * Get the allowed maximum number of RX queues.
1067  * *pid return the port id which has minimal value of
1068  * max_rx_queues in all ports.
1069  */
1070 queueid_t
1071 get_allowed_max_nb_rxq(portid_t *pid)
1072 {
1073         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1074         bool max_rxq_valid = false;
1075         portid_t pi;
1076         struct rte_eth_dev_info dev_info;
1077
1078         RTE_ETH_FOREACH_DEV(pi) {
1079                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1080                         continue;
1081
1082                 max_rxq_valid = true;
1083                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1084                         allowed_max_rxq = dev_info.max_rx_queues;
1085                         *pid = pi;
1086                 }
1087         }
1088         return max_rxq_valid ? allowed_max_rxq : 0;
1089 }
1090
1091 /*
1092  * Check input rxq is valid or not.
1093  * If input rxq is not greater than any of maximum number
1094  * of RX queues of all ports, it is valid.
1095  * if valid, return 0, else return -1
1096  */
1097 int
1098 check_nb_rxq(queueid_t rxq)
1099 {
1100         queueid_t allowed_max_rxq;
1101         portid_t pid = 0;
1102
1103         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1104         if (rxq > allowed_max_rxq) {
1105                 printf("Fail: input rxq (%u) can't be greater "
1106                        "than max_rx_queues (%u) of port %u\n",
1107                        rxq,
1108                        allowed_max_rxq,
1109                        pid);
1110                 return -1;
1111         }
1112         return 0;
1113 }
1114
1115 /*
1116  * Get the allowed maximum number of TX queues.
1117  * *pid return the port id which has minimal value of
1118  * max_tx_queues in all ports.
1119  */
1120 queueid_t
1121 get_allowed_max_nb_txq(portid_t *pid)
1122 {
1123         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1124         bool max_txq_valid = false;
1125         portid_t pi;
1126         struct rte_eth_dev_info dev_info;
1127
1128         RTE_ETH_FOREACH_DEV(pi) {
1129                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1130                         continue;
1131
1132                 max_txq_valid = true;
1133                 if (dev_info.max_tx_queues < allowed_max_txq) {
1134                         allowed_max_txq = dev_info.max_tx_queues;
1135                         *pid = pi;
1136                 }
1137         }
1138         return max_txq_valid ? allowed_max_txq : 0;
1139 }
1140
1141 /*
1142  * Check input txq is valid or not.
1143  * If input txq is not greater than any of maximum number
1144  * of TX queues of all ports, it is valid.
1145  * if valid, return 0, else return -1
1146  */
1147 int
1148 check_nb_txq(queueid_t txq)
1149 {
1150         queueid_t allowed_max_txq;
1151         portid_t pid = 0;
1152
1153         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1154         if (txq > allowed_max_txq) {
1155                 printf("Fail: input txq (%u) can't be greater "
1156                        "than max_tx_queues (%u) of port %u\n",
1157                        txq,
1158                        allowed_max_txq,
1159                        pid);
1160                 return -1;
1161         }
1162         return 0;
1163 }
1164
1165 /*
1166  * Get the allowed maximum number of RXDs of every rx queue.
1167  * *pid return the port id which has minimal value of
1168  * max_rxd in all queues of all ports.
1169  */
1170 static uint16_t
1171 get_allowed_max_nb_rxd(portid_t *pid)
1172 {
1173         uint16_t allowed_max_rxd = UINT16_MAX;
1174         portid_t pi;
1175         struct rte_eth_dev_info dev_info;
1176
1177         RTE_ETH_FOREACH_DEV(pi) {
1178                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1179                         continue;
1180
1181                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1182                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1183                         *pid = pi;
1184                 }
1185         }
1186         return allowed_max_rxd;
1187 }
1188
1189 /*
1190  * Get the allowed minimal number of RXDs of every rx queue.
1191  * *pid return the port id which has minimal value of
1192  * min_rxd in all queues of all ports.
1193  */
1194 static uint16_t
1195 get_allowed_min_nb_rxd(portid_t *pid)
1196 {
1197         uint16_t allowed_min_rxd = 0;
1198         portid_t pi;
1199         struct rte_eth_dev_info dev_info;
1200
1201         RTE_ETH_FOREACH_DEV(pi) {
1202                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1203                         continue;
1204
1205                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1206                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1207                         *pid = pi;
1208                 }
1209         }
1210
1211         return allowed_min_rxd;
1212 }
1213
1214 /*
1215  * Check input rxd is valid or not.
1216  * If input rxd is not greater than any of maximum number
1217  * of RXDs of every Rx queues and is not less than any of
1218  * minimal number of RXDs of every Rx queues, it is valid.
1219  * if valid, return 0, else return -1
1220  */
1221 int
1222 check_nb_rxd(queueid_t rxd)
1223 {
1224         uint16_t allowed_max_rxd;
1225         uint16_t allowed_min_rxd;
1226         portid_t pid = 0;
1227
1228         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1229         if (rxd > allowed_max_rxd) {
1230                 printf("Fail: input rxd (%u) can't be greater "
1231                        "than max_rxds (%u) of port %u\n",
1232                        rxd,
1233                        allowed_max_rxd,
1234                        pid);
1235                 return -1;
1236         }
1237
1238         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1239         if (rxd < allowed_min_rxd) {
1240                 printf("Fail: input rxd (%u) can't be less "
1241                        "than min_rxds (%u) of port %u\n",
1242                        rxd,
1243                        allowed_min_rxd,
1244                        pid);
1245                 return -1;
1246         }
1247
1248         return 0;
1249 }
1250
1251 /*
1252  * Get the allowed maximum number of TXDs of every rx queues.
1253  * *pid return the port id which has minimal value of
1254  * max_txd in every tx queue.
1255  */
1256 static uint16_t
1257 get_allowed_max_nb_txd(portid_t *pid)
1258 {
1259         uint16_t allowed_max_txd = UINT16_MAX;
1260         portid_t pi;
1261         struct rte_eth_dev_info dev_info;
1262
1263         RTE_ETH_FOREACH_DEV(pi) {
1264                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1265                         continue;
1266
1267                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1268                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1269                         *pid = pi;
1270                 }
1271         }
1272         return allowed_max_txd;
1273 }
1274
1275 /*
1276  * Get the allowed maximum number of TXDs of every tx queues.
1277  * *pid return the port id which has minimal value of
1278  * min_txd in every tx queue.
1279  */
1280 static uint16_t
1281 get_allowed_min_nb_txd(portid_t *pid)
1282 {
1283         uint16_t allowed_min_txd = 0;
1284         portid_t pi;
1285         struct rte_eth_dev_info dev_info;
1286
1287         RTE_ETH_FOREACH_DEV(pi) {
1288                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1289                         continue;
1290
1291                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1292                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1293                         *pid = pi;
1294                 }
1295         }
1296
1297         return allowed_min_txd;
1298 }
1299
1300 /*
1301  * Check input txd is valid or not.
1302  * If input txd is not greater than any of maximum number
1303  * of TXDs of every Rx queues, it is valid.
1304  * if valid, return 0, else return -1
1305  */
1306 int
1307 check_nb_txd(queueid_t txd)
1308 {
1309         uint16_t allowed_max_txd;
1310         uint16_t allowed_min_txd;
1311         portid_t pid = 0;
1312
1313         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1314         if (txd > allowed_max_txd) {
1315                 printf("Fail: input txd (%u) can't be greater "
1316                        "than max_txds (%u) of port %u\n",
1317                        txd,
1318                        allowed_max_txd,
1319                        pid);
1320                 return -1;
1321         }
1322
1323         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1324         if (txd < allowed_min_txd) {
1325                 printf("Fail: input txd (%u) can't be less "
1326                        "than min_txds (%u) of port %u\n",
1327                        txd,
1328                        allowed_min_txd,
1329                        pid);
1330                 return -1;
1331         }
1332         return 0;
1333 }
1334
1335
1336 /*
1337  * Get the allowed maximum number of hairpin queues.
1338  * *pid return the port id which has minimal value of
1339  * max_hairpin_queues in all ports.
1340  */
1341 queueid_t
1342 get_allowed_max_nb_hairpinq(portid_t *pid)
1343 {
1344         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1345         portid_t pi;
1346         struct rte_eth_hairpin_cap cap;
1347
1348         RTE_ETH_FOREACH_DEV(pi) {
1349                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1350                         *pid = pi;
1351                         return 0;
1352                 }
1353                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1354                         allowed_max_hairpinq = cap.max_nb_queues;
1355                         *pid = pi;
1356                 }
1357         }
1358         return allowed_max_hairpinq;
1359 }
1360
1361 /*
1362  * Check input hairpin is valid or not.
1363  * If input hairpin is not greater than any of maximum number
1364  * of hairpin queues of all ports, it is valid.
1365  * if valid, return 0, else return -1
1366  */
1367 int
1368 check_nb_hairpinq(queueid_t hairpinq)
1369 {
1370         queueid_t allowed_max_hairpinq;
1371         portid_t pid = 0;
1372
1373         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1374         if (hairpinq > allowed_max_hairpinq) {
1375                 printf("Fail: input hairpin (%u) can't be greater "
1376                        "than max_hairpin_queues (%u) of port %u\n",
1377                        hairpinq, allowed_max_hairpinq, pid);
1378                 return -1;
1379         }
1380         return 0;
1381 }
1382
1383 static void
1384 init_config(void)
1385 {
1386         portid_t pid;
1387         struct rte_port *port;
1388         struct rte_mempool *mbp;
1389         unsigned int nb_mbuf_per_pool;
1390         lcoreid_t  lc_id;
1391         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1392         struct rte_gro_param gro_param;
1393         uint32_t gso_types;
1394         uint16_t data_size;
1395         bool warning = 0;
1396         int k;
1397         int ret;
1398
1399         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1400
1401         /* Configuration of logical cores. */
1402         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1403                                 sizeof(struct fwd_lcore *) * nb_lcores,
1404                                 RTE_CACHE_LINE_SIZE);
1405         if (fwd_lcores == NULL) {
1406                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1407                                                         "failed\n", nb_lcores);
1408         }
1409         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1410                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1411                                                sizeof(struct fwd_lcore),
1412                                                RTE_CACHE_LINE_SIZE);
1413                 if (fwd_lcores[lc_id] == NULL) {
1414                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1415                                                                 "failed\n");
1416                 }
1417                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1418         }
1419
1420         RTE_ETH_FOREACH_DEV(pid) {
1421                 port = &ports[pid];
1422                 /* Apply default TxRx configuration for all ports */
1423                 port->dev_conf.txmode = tx_mode;
1424                 port->dev_conf.rxmode = rx_mode;
1425
1426                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1427                 if (ret != 0)
1428                         rte_exit(EXIT_FAILURE,
1429                                  "rte_eth_dev_info_get() failed\n");
1430
1431                 if (!(port->dev_info.tx_offload_capa &
1432                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1433                         port->dev_conf.txmode.offloads &=
1434                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1435                 if (numa_support) {
1436                         if (port_numa[pid] != NUMA_NO_CONFIG)
1437                                 port_per_socket[port_numa[pid]]++;
1438                         else {
1439                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1440
1441                                 /*
1442                                  * if socket_id is invalid,
1443                                  * set to the first available socket.
1444                                  */
1445                                 if (check_socket_id(socket_id) < 0)
1446                                         socket_id = socket_ids[0];
1447                                 port_per_socket[socket_id]++;
1448                         }
1449                 }
1450
1451                 /* Apply Rx offloads configuration */
1452                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1453                         port->rx_conf[k].offloads =
1454                                 port->dev_conf.rxmode.offloads;
1455                 /* Apply Tx offloads configuration */
1456                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1457                         port->tx_conf[k].offloads =
1458                                 port->dev_conf.txmode.offloads;
1459
1460                 /* set flag to initialize port/queue */
1461                 port->need_reconfig = 1;
1462                 port->need_reconfig_queues = 1;
1463                 port->tx_metadata = 0;
1464
1465                 /* Check for maximum number of segments per MTU. Accordingly
1466                  * update the mbuf data size.
1467                  */
1468                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1469                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1470                         data_size = rx_mode.max_rx_pkt_len /
1471                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1472
1473                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1474                                                         mbuf_data_size) {
1475                                 mbuf_data_size = data_size +
1476                                                  RTE_PKTMBUF_HEADROOM;
1477                                 warning = 1;
1478                         }
1479                 }
1480         }
1481
1482         if (warning)
1483                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1484                             mbuf_data_size);
1485
1486         /*
1487          * Create pools of mbuf.
1488          * If NUMA support is disabled, create a single pool of mbuf in
1489          * socket 0 memory by default.
1490          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1491          *
1492          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1493          * nb_txd can be configured at run time.
1494          */
1495         if (param_total_num_mbufs)
1496                 nb_mbuf_per_pool = param_total_num_mbufs;
1497         else {
1498                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1499                         (nb_lcores * mb_mempool_cache) +
1500                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1501                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1502         }
1503
1504         if (numa_support) {
1505                 uint8_t i;
1506
1507                 for (i = 0; i < num_sockets; i++)
1508                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1509                                                        nb_mbuf_per_pool,
1510                                                        socket_ids[i]);
1511         } else {
1512                 if (socket_num == UMA_NO_CONFIG)
1513                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1514                                                        nb_mbuf_per_pool, 0);
1515                 else
1516                         mempools[socket_num] = mbuf_pool_create
1517                                                         (mbuf_data_size,
1518                                                          nb_mbuf_per_pool,
1519                                                          socket_num);
1520         }
1521
1522         init_port_config();
1523
1524         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1525                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1526         /*
1527          * Records which Mbuf pool to use by each logical core, if needed.
1528          */
1529         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1530                 mbp = mbuf_pool_find(
1531                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1532
1533                 if (mbp == NULL)
1534                         mbp = mbuf_pool_find(0);
1535                 fwd_lcores[lc_id]->mbp = mbp;
1536                 /* initialize GSO context */
1537                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1538                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1539                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1540                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1541                         RTE_ETHER_CRC_LEN;
1542                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1543         }
1544
1545         /* Configuration of packet forwarding streams. */
1546         if (init_fwd_streams() < 0)
1547                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1548
1549         fwd_config_setup();
1550
1551         /* create a gro context for each lcore */
1552         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1553         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1554         gro_param.max_item_per_flow = MAX_PKT_BURST;
1555         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1556                 gro_param.socket_id = rte_lcore_to_socket_id(
1557                                 fwd_lcores_cpuids[lc_id]);
1558                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1559                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1560                         rte_exit(EXIT_FAILURE,
1561                                         "rte_gro_ctx_create() failed\n");
1562                 }
1563         }
1564
1565 #if defined RTE_LIBRTE_PMD_SOFTNIC
1566         if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) {
1567                 RTE_ETH_FOREACH_DEV(pid) {
1568                         port = &ports[pid];
1569                         const char *driver = port->dev_info.driver_name;
1570
1571                         if (strcmp(driver, "net_softnic") == 0)
1572                                 port->softport.fwd_lcore_arg = fwd_lcores;
1573                 }
1574         }
1575 #endif
1576
1577 }
1578
1579
1580 void
1581 reconfig(portid_t new_port_id, unsigned socket_id)
1582 {
1583         struct rte_port *port;
1584         int ret;
1585
1586         /* Reconfiguration of Ethernet ports. */
1587         port = &ports[new_port_id];
1588
1589         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1590         if (ret != 0)
1591                 return;
1592
1593         /* set flag to initialize port/queue */
1594         port->need_reconfig = 1;
1595         port->need_reconfig_queues = 1;
1596         port->socket_id = socket_id;
1597
1598         init_port_config();
1599 }
1600
1601
1602 int
1603 init_fwd_streams(void)
1604 {
1605         portid_t pid;
1606         struct rte_port *port;
1607         streamid_t sm_id, nb_fwd_streams_new;
1608         queueid_t q;
1609
1610         /* set socket id according to numa or not */
1611         RTE_ETH_FOREACH_DEV(pid) {
1612                 port = &ports[pid];
1613                 if (nb_rxq > port->dev_info.max_rx_queues) {
1614                         printf("Fail: nb_rxq(%d) is greater than "
1615                                 "max_rx_queues(%d)\n", nb_rxq,
1616                                 port->dev_info.max_rx_queues);
1617                         return -1;
1618                 }
1619                 if (nb_txq > port->dev_info.max_tx_queues) {
1620                         printf("Fail: nb_txq(%d) is greater than "
1621                                 "max_tx_queues(%d)\n", nb_txq,
1622                                 port->dev_info.max_tx_queues);
1623                         return -1;
1624                 }
1625                 if (numa_support) {
1626                         if (port_numa[pid] != NUMA_NO_CONFIG)
1627                                 port->socket_id = port_numa[pid];
1628                         else {
1629                                 port->socket_id = rte_eth_dev_socket_id(pid);
1630
1631                                 /*
1632                                  * if socket_id is invalid,
1633                                  * set to the first available socket.
1634                                  */
1635                                 if (check_socket_id(port->socket_id) < 0)
1636                                         port->socket_id = socket_ids[0];
1637                         }
1638                 }
1639                 else {
1640                         if (socket_num == UMA_NO_CONFIG)
1641                                 port->socket_id = 0;
1642                         else
1643                                 port->socket_id = socket_num;
1644                 }
1645         }
1646
1647         q = RTE_MAX(nb_rxq, nb_txq);
1648         if (q == 0) {
1649                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1650                 return -1;
1651         }
1652         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1653         if (nb_fwd_streams_new == nb_fwd_streams)
1654                 return 0;
1655         /* clear the old */
1656         if (fwd_streams != NULL) {
1657                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1658                         if (fwd_streams[sm_id] == NULL)
1659                                 continue;
1660                         rte_free(fwd_streams[sm_id]);
1661                         fwd_streams[sm_id] = NULL;
1662                 }
1663                 rte_free(fwd_streams);
1664                 fwd_streams = NULL;
1665         }
1666
1667         /* init new */
1668         nb_fwd_streams = nb_fwd_streams_new;
1669         if (nb_fwd_streams) {
1670                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1671                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1672                         RTE_CACHE_LINE_SIZE);
1673                 if (fwd_streams == NULL)
1674                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1675                                  " (struct fwd_stream *)) failed\n",
1676                                  nb_fwd_streams);
1677
1678                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1679                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1680                                 " struct fwd_stream", sizeof(struct fwd_stream),
1681                                 RTE_CACHE_LINE_SIZE);
1682                         if (fwd_streams[sm_id] == NULL)
1683                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1684                                          "(struct fwd_stream) failed\n");
1685                 }
1686         }
1687
1688         return 0;
1689 }
1690
1691 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1692 static void
1693 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1694 {
1695         unsigned int total_burst;
1696         unsigned int nb_burst;
1697         unsigned int burst_stats[3];
1698         uint16_t pktnb_stats[3];
1699         uint16_t nb_pkt;
1700         int burst_percent[3];
1701
1702         /*
1703          * First compute the total number of packet bursts and the
1704          * two highest numbers of bursts of the same number of packets.
1705          */
1706         total_burst = 0;
1707         burst_stats[0] = burst_stats[1] = burst_stats[2] = 0;
1708         pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0;
1709         for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1710                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1711                 if (nb_burst == 0)
1712                         continue;
1713                 total_burst += nb_burst;
1714                 if (nb_burst > burst_stats[0]) {
1715                         burst_stats[1] = burst_stats[0];
1716                         pktnb_stats[1] = pktnb_stats[0];
1717                         burst_stats[0] = nb_burst;
1718                         pktnb_stats[0] = nb_pkt;
1719                 } else if (nb_burst > burst_stats[1]) {
1720                         burst_stats[1] = nb_burst;
1721                         pktnb_stats[1] = nb_pkt;
1722                 }
1723         }
1724         if (total_burst == 0)
1725                 return;
1726         burst_percent[0] = (burst_stats[0] * 100) / total_burst;
1727         printf("  %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst,
1728                burst_percent[0], (int) pktnb_stats[0]);
1729         if (burst_stats[0] == total_burst) {
1730                 printf("]\n");
1731                 return;
1732         }
1733         if (burst_stats[0] + burst_stats[1] == total_burst) {
1734                 printf(" + %d%% of %d pkts]\n",
1735                        100 - burst_percent[0], pktnb_stats[1]);
1736                 return;
1737         }
1738         burst_percent[1] = (burst_stats[1] * 100) / total_burst;
1739         burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]);
1740         if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) {
1741                 printf(" + %d%% of others]\n", 100 - burst_percent[0]);
1742                 return;
1743         }
1744         printf(" + %d%% of %d pkts + %d%% of others]\n",
1745                burst_percent[1], (int) pktnb_stats[1], burst_percent[2]);
1746 }
1747 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1748
1749 static void
1750 fwd_stream_stats_display(streamid_t stream_id)
1751 {
1752         struct fwd_stream *fs;
1753         static const char *fwd_top_stats_border = "-------";
1754
1755         fs = fwd_streams[stream_id];
1756         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1757             (fs->fwd_dropped == 0))
1758                 return;
1759         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1760                "TX Port=%2d/Queue=%2d %s\n",
1761                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1762                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1763         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1764                " TX-dropped: %-14"PRIu64,
1765                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1766
1767         /* if checksum mode */
1768         if (cur_fwd_eng == &csum_fwd_engine) {
1769                 printf("  RX- bad IP checksum: %-14"PRIu64
1770                        "  Rx- bad L4 checksum: %-14"PRIu64
1771                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1772                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1773                         fs->rx_bad_outer_l4_csum);
1774         } else {
1775                 printf("\n");
1776         }
1777
1778 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1779         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1780         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1781 #endif
1782 }
1783
1784 void
1785 fwd_stats_display(void)
1786 {
1787         static const char *fwd_stats_border = "----------------------";
1788         static const char *acc_stats_border = "+++++++++++++++";
1789         struct {
1790                 struct fwd_stream *rx_stream;
1791                 struct fwd_stream *tx_stream;
1792                 uint64_t tx_dropped;
1793                 uint64_t rx_bad_ip_csum;
1794                 uint64_t rx_bad_l4_csum;
1795                 uint64_t rx_bad_outer_l4_csum;
1796         } ports_stats[RTE_MAX_ETHPORTS];
1797         uint64_t total_rx_dropped = 0;
1798         uint64_t total_tx_dropped = 0;
1799         uint64_t total_rx_nombuf = 0;
1800         struct rte_eth_stats stats;
1801 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1802         uint64_t fwd_cycles = 0;
1803 #endif
1804         uint64_t total_recv = 0;
1805         uint64_t total_xmit = 0;
1806         struct rte_port *port;
1807         streamid_t sm_id;
1808         portid_t pt_id;
1809         int i;
1810
1811         memset(ports_stats, 0, sizeof(ports_stats));
1812
1813         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1814                 struct fwd_stream *fs = fwd_streams[sm_id];
1815
1816                 if (cur_fwd_config.nb_fwd_streams >
1817                     cur_fwd_config.nb_fwd_ports) {
1818                         fwd_stream_stats_display(sm_id);
1819                 } else {
1820                         ports_stats[fs->tx_port].tx_stream = fs;
1821                         ports_stats[fs->rx_port].rx_stream = fs;
1822                 }
1823
1824                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1825
1826                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1827                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1828                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1829                                 fs->rx_bad_outer_l4_csum;
1830
1831 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1832                 fwd_cycles += fs->core_cycles;
1833 #endif
1834         }
1835         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1836                 uint8_t j;
1837
1838                 pt_id = fwd_ports_ids[i];
1839                 port = &ports[pt_id];
1840
1841                 rte_eth_stats_get(pt_id, &stats);
1842                 stats.ipackets -= port->stats.ipackets;
1843                 stats.opackets -= port->stats.opackets;
1844                 stats.ibytes -= port->stats.ibytes;
1845                 stats.obytes -= port->stats.obytes;
1846                 stats.imissed -= port->stats.imissed;
1847                 stats.oerrors -= port->stats.oerrors;
1848                 stats.rx_nombuf -= port->stats.rx_nombuf;
1849
1850                 total_recv += stats.ipackets;
1851                 total_xmit += stats.opackets;
1852                 total_rx_dropped += stats.imissed;
1853                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1854                 total_tx_dropped += stats.oerrors;
1855                 total_rx_nombuf  += stats.rx_nombuf;
1856
1857                 printf("\n  %s Forward statistics for port %-2d %s\n",
1858                        fwd_stats_border, pt_id, fwd_stats_border);
1859
1860                 if (!port->rx_queue_stats_mapping_enabled &&
1861                     !port->tx_queue_stats_mapping_enabled) {
1862                         printf("  RX-packets: %-14"PRIu64
1863                                " RX-dropped: %-14"PRIu64
1864                                "RX-total: %-"PRIu64"\n",
1865                                stats.ipackets, stats.imissed,
1866                                stats.ipackets + stats.imissed);
1867
1868                         if (cur_fwd_eng == &csum_fwd_engine)
1869                                 printf("  Bad-ipcsum: %-14"PRIu64
1870                                        " Bad-l4csum: %-14"PRIu64
1871                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1872                                        ports_stats[pt_id].rx_bad_ip_csum,
1873                                        ports_stats[pt_id].rx_bad_l4_csum,
1874                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1875                         if (stats.ierrors + stats.rx_nombuf > 0) {
1876                                 printf("  RX-error: %-"PRIu64"\n",
1877                                        stats.ierrors);
1878                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1879                                        stats.rx_nombuf);
1880                         }
1881
1882                         printf("  TX-packets: %-14"PRIu64
1883                                " TX-dropped: %-14"PRIu64
1884                                "TX-total: %-"PRIu64"\n",
1885                                stats.opackets, ports_stats[pt_id].tx_dropped,
1886                                stats.opackets + ports_stats[pt_id].tx_dropped);
1887                 } else {
1888                         printf("  RX-packets:             %14"PRIu64
1889                                "    RX-dropped:%14"PRIu64
1890                                "    RX-total:%14"PRIu64"\n",
1891                                stats.ipackets, stats.imissed,
1892                                stats.ipackets + stats.imissed);
1893
1894                         if (cur_fwd_eng == &csum_fwd_engine)
1895                                 printf("  Bad-ipcsum:%14"PRIu64
1896                                        "    Bad-l4csum:%14"PRIu64
1897                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1898                                        ports_stats[pt_id].rx_bad_ip_csum,
1899                                        ports_stats[pt_id].rx_bad_l4_csum,
1900                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1901                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1902                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1903                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1904                                        stats.rx_nombuf);
1905                         }
1906
1907                         printf("  TX-packets:             %14"PRIu64
1908                                "    TX-dropped:%14"PRIu64
1909                                "    TX-total:%14"PRIu64"\n",
1910                                stats.opackets, ports_stats[pt_id].tx_dropped,
1911                                stats.opackets + ports_stats[pt_id].tx_dropped);
1912                 }
1913
1914 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1915                 if (ports_stats[pt_id].rx_stream)
1916                         pkt_burst_stats_display("RX",
1917                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1918                 if (ports_stats[pt_id].tx_stream)
1919                         pkt_burst_stats_display("TX",
1920                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1921 #endif
1922
1923                 if (port->rx_queue_stats_mapping_enabled) {
1924                         printf("\n");
1925                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1926                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1927                                        "     RX-errors:%14"PRIu64
1928                                        "    RX-bytes:%14"PRIu64"\n",
1929                                        j, stats.q_ipackets[j],
1930                                        stats.q_errors[j], stats.q_ibytes[j]);
1931                         }
1932                         printf("\n");
1933                 }
1934                 if (port->tx_queue_stats_mapping_enabled) {
1935                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1936                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1937                                        "                                 TX-bytes:%14"
1938                                        PRIu64"\n",
1939                                        j, stats.q_opackets[j],
1940                                        stats.q_obytes[j]);
1941                         }
1942                 }
1943
1944                 printf("  %s--------------------------------%s\n",
1945                        fwd_stats_border, fwd_stats_border);
1946         }
1947
1948         printf("\n  %s Accumulated forward statistics for all ports"
1949                "%s\n",
1950                acc_stats_border, acc_stats_border);
1951         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1952                "%-"PRIu64"\n"
1953                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1954                "%-"PRIu64"\n",
1955                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1956                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1957         if (total_rx_nombuf > 0)
1958                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1959         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1960                "%s\n",
1961                acc_stats_border, acc_stats_border);
1962 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1963         if (total_recv > 0)
1964                 printf("\n  CPU cycles/packet=%u (total cycles="
1965                        "%"PRIu64" / total RX packets=%"PRIu64")\n",
1966                        (unsigned int)(fwd_cycles / total_recv),
1967                        fwd_cycles, total_recv);
1968 #endif
1969 }
1970
1971 void
1972 fwd_stats_reset(void)
1973 {
1974         streamid_t sm_id;
1975         portid_t pt_id;
1976         int i;
1977
1978         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1979                 pt_id = fwd_ports_ids[i];
1980                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1981         }
1982         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1983                 struct fwd_stream *fs = fwd_streams[sm_id];
1984
1985                 fs->rx_packets = 0;
1986                 fs->tx_packets = 0;
1987                 fs->fwd_dropped = 0;
1988                 fs->rx_bad_ip_csum = 0;
1989                 fs->rx_bad_l4_csum = 0;
1990                 fs->rx_bad_outer_l4_csum = 0;
1991
1992 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1993                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1994                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1995 #endif
1996 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1997                 fs->core_cycles = 0;
1998 #endif
1999         }
2000 }
2001
2002 static void
2003 flush_fwd_rx_queues(void)
2004 {
2005         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2006         portid_t  rxp;
2007         portid_t port_id;
2008         queueid_t rxq;
2009         uint16_t  nb_rx;
2010         uint16_t  i;
2011         uint8_t   j;
2012         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2013         uint64_t timer_period;
2014
2015         /* convert to number of cycles */
2016         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2017
2018         for (j = 0; j < 2; j++) {
2019                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2020                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2021                                 port_id = fwd_ports_ids[rxp];
2022                                 /**
2023                                 * testpmd can stuck in the below do while loop
2024                                 * if rte_eth_rx_burst() always returns nonzero
2025                                 * packets. So timer is added to exit this loop
2026                                 * after 1sec timer expiry.
2027                                 */
2028                                 prev_tsc = rte_rdtsc();
2029                                 do {
2030                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2031                                                 pkts_burst, MAX_PKT_BURST);
2032                                         for (i = 0; i < nb_rx; i++)
2033                                                 rte_pktmbuf_free(pkts_burst[i]);
2034
2035                                         cur_tsc = rte_rdtsc();
2036                                         diff_tsc = cur_tsc - prev_tsc;
2037                                         timer_tsc += diff_tsc;
2038                                 } while ((nb_rx > 0) &&
2039                                         (timer_tsc < timer_period));
2040                                 timer_tsc = 0;
2041                         }
2042                 }
2043                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2044         }
2045 }
2046
2047 static void
2048 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2049 {
2050         struct fwd_stream **fsm;
2051         streamid_t nb_fs;
2052         streamid_t sm_id;
2053 #ifdef RTE_LIBRTE_BITRATE
2054         uint64_t tics_per_1sec;
2055         uint64_t tics_datum;
2056         uint64_t tics_current;
2057         uint16_t i, cnt_ports;
2058
2059         cnt_ports = nb_ports;
2060         tics_datum = rte_rdtsc();
2061         tics_per_1sec = rte_get_timer_hz();
2062 #endif
2063         fsm = &fwd_streams[fc->stream_idx];
2064         nb_fs = fc->stream_nb;
2065         do {
2066                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2067                         (*pkt_fwd)(fsm[sm_id]);
2068 #ifdef RTE_LIBRTE_BITRATE
2069                 if (bitrate_enabled != 0 &&
2070                                 bitrate_lcore_id == rte_lcore_id()) {
2071                         tics_current = rte_rdtsc();
2072                         if (tics_current - tics_datum >= tics_per_1sec) {
2073                                 /* Periodic bitrate calculation */
2074                                 for (i = 0; i < cnt_ports; i++)
2075                                         rte_stats_bitrate_calc(bitrate_data,
2076                                                 ports_ids[i]);
2077                                 tics_datum = tics_current;
2078                         }
2079                 }
2080 #endif
2081 #ifdef RTE_LIBRTE_LATENCY_STATS
2082                 if (latencystats_enabled != 0 &&
2083                                 latencystats_lcore_id == rte_lcore_id())
2084                         rte_latencystats_update();
2085 #endif
2086
2087         } while (! fc->stopped);
2088 }
2089
2090 static int
2091 start_pkt_forward_on_core(void *fwd_arg)
2092 {
2093         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2094                              cur_fwd_config.fwd_eng->packet_fwd);
2095         return 0;
2096 }
2097
2098 /*
2099  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2100  * Used to start communication flows in network loopback test configurations.
2101  */
2102 static int
2103 run_one_txonly_burst_on_core(void *fwd_arg)
2104 {
2105         struct fwd_lcore *fwd_lc;
2106         struct fwd_lcore tmp_lcore;
2107
2108         fwd_lc = (struct fwd_lcore *) fwd_arg;
2109         tmp_lcore = *fwd_lc;
2110         tmp_lcore.stopped = 1;
2111         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2112         return 0;
2113 }
2114
2115 /*
2116  * Launch packet forwarding:
2117  *     - Setup per-port forwarding context.
2118  *     - launch logical cores with their forwarding configuration.
2119  */
2120 static void
2121 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2122 {
2123         port_fwd_begin_t port_fwd_begin;
2124         unsigned int i;
2125         unsigned int lc_id;
2126         int diag;
2127
2128         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2129         if (port_fwd_begin != NULL) {
2130                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2131                         (*port_fwd_begin)(fwd_ports_ids[i]);
2132         }
2133         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2134                 lc_id = fwd_lcores_cpuids[i];
2135                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2136                         fwd_lcores[i]->stopped = 0;
2137                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2138                                                      fwd_lcores[i], lc_id);
2139                         if (diag != 0)
2140                                 printf("launch lcore %u failed - diag=%d\n",
2141                                        lc_id, diag);
2142                 }
2143         }
2144 }
2145
2146 /*
2147  * Launch packet forwarding configuration.
2148  */
2149 void
2150 start_packet_forwarding(int with_tx_first)
2151 {
2152         port_fwd_begin_t port_fwd_begin;
2153         port_fwd_end_t  port_fwd_end;
2154         struct rte_port *port;
2155         unsigned int i;
2156         portid_t   pt_id;
2157
2158         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2159                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2160
2161         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2162                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2163
2164         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2165                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2166                 (!nb_rxq || !nb_txq))
2167                 rte_exit(EXIT_FAILURE,
2168                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2169                         cur_fwd_eng->fwd_mode_name);
2170
2171         if (all_ports_started() == 0) {
2172                 printf("Not all ports were started\n");
2173                 return;
2174         }
2175         if (test_done == 0) {
2176                 printf("Packet forwarding already started\n");
2177                 return;
2178         }
2179
2180
2181         if(dcb_test) {
2182                 for (i = 0; i < nb_fwd_ports; i++) {
2183                         pt_id = fwd_ports_ids[i];
2184                         port = &ports[pt_id];
2185                         if (!port->dcb_flag) {
2186                                 printf("In DCB mode, all forwarding ports must "
2187                                        "be configured in this mode.\n");
2188                                 return;
2189                         }
2190                 }
2191                 if (nb_fwd_lcores == 1) {
2192                         printf("In DCB mode,the nb forwarding cores "
2193                                "should be larger than 1.\n");
2194                         return;
2195                 }
2196         }
2197         test_done = 0;
2198
2199         fwd_config_setup();
2200
2201         if(!no_flush_rx)
2202                 flush_fwd_rx_queues();
2203
2204         pkt_fwd_config_display(&cur_fwd_config);
2205         rxtx_config_display();
2206
2207         fwd_stats_reset();
2208         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2209                 pt_id = fwd_ports_ids[i];
2210                 port = &ports[pt_id];
2211                 map_port_queue_stats_mapping_registers(pt_id, port);
2212         }
2213         if (with_tx_first) {
2214                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2215                 if (port_fwd_begin != NULL) {
2216                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2217                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2218                 }
2219                 while (with_tx_first--) {
2220                         launch_packet_forwarding(
2221                                         run_one_txonly_burst_on_core);
2222                         rte_eal_mp_wait_lcore();
2223                 }
2224                 port_fwd_end = tx_only_engine.port_fwd_end;
2225                 if (port_fwd_end != NULL) {
2226                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2227                                 (*port_fwd_end)(fwd_ports_ids[i]);
2228                 }
2229         }
2230         launch_packet_forwarding(start_pkt_forward_on_core);
2231 }
2232
2233 void
2234 stop_packet_forwarding(void)
2235 {
2236         port_fwd_end_t port_fwd_end;
2237         lcoreid_t lc_id;
2238         portid_t pt_id;
2239         int i;
2240
2241         if (test_done) {
2242                 printf("Packet forwarding not started\n");
2243                 return;
2244         }
2245         printf("Telling cores to stop...");
2246         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2247                 fwd_lcores[lc_id]->stopped = 1;
2248         printf("\nWaiting for lcores to finish...\n");
2249         rte_eal_mp_wait_lcore();
2250         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2251         if (port_fwd_end != NULL) {
2252                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2253                         pt_id = fwd_ports_ids[i];
2254                         (*port_fwd_end)(pt_id);
2255                 }
2256         }
2257
2258         fwd_stats_display();
2259
2260         printf("\nDone.\n");
2261         test_done = 1;
2262 }
2263
2264 void
2265 dev_set_link_up(portid_t pid)
2266 {
2267         if (rte_eth_dev_set_link_up(pid) < 0)
2268                 printf("\nSet link up fail.\n");
2269 }
2270
2271 void
2272 dev_set_link_down(portid_t pid)
2273 {
2274         if (rte_eth_dev_set_link_down(pid) < 0)
2275                 printf("\nSet link down fail.\n");
2276 }
2277
2278 static int
2279 all_ports_started(void)
2280 {
2281         portid_t pi;
2282         struct rte_port *port;
2283
2284         RTE_ETH_FOREACH_DEV(pi) {
2285                 port = &ports[pi];
2286                 /* Check if there is a port which is not started */
2287                 if ((port->port_status != RTE_PORT_STARTED) &&
2288                         (port->slave_flag == 0))
2289                         return 0;
2290         }
2291
2292         /* No port is not started */
2293         return 1;
2294 }
2295
2296 int
2297 port_is_stopped(portid_t port_id)
2298 {
2299         struct rte_port *port = &ports[port_id];
2300
2301         if ((port->port_status != RTE_PORT_STOPPED) &&
2302             (port->slave_flag == 0))
2303                 return 0;
2304         return 1;
2305 }
2306
2307 int
2308 all_ports_stopped(void)
2309 {
2310         portid_t pi;
2311
2312         RTE_ETH_FOREACH_DEV(pi) {
2313                 if (!port_is_stopped(pi))
2314                         return 0;
2315         }
2316
2317         return 1;
2318 }
2319
2320 int
2321 port_is_started(portid_t port_id)
2322 {
2323         if (port_id_is_invalid(port_id, ENABLED_WARN))
2324                 return 0;
2325
2326         if (ports[port_id].port_status != RTE_PORT_STARTED)
2327                 return 0;
2328
2329         return 1;
2330 }
2331
2332 /* Configure the Rx and Tx hairpin queues for the selected port. */
2333 static int
2334 setup_hairpin_queues(portid_t pi)
2335 {
2336         queueid_t qi;
2337         struct rte_eth_hairpin_conf hairpin_conf = {
2338                 .peer_count = 1,
2339         };
2340         int i;
2341         int diag;
2342         struct rte_port *port = &ports[pi];
2343
2344         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2345                 hairpin_conf.peers[0].port = pi;
2346                 hairpin_conf.peers[0].queue = i + nb_rxq;
2347                 diag = rte_eth_tx_hairpin_queue_setup
2348                         (pi, qi, nb_txd, &hairpin_conf);
2349                 i++;
2350                 if (diag == 0)
2351                         continue;
2352
2353                 /* Fail to setup rx queue, return */
2354                 if (rte_atomic16_cmpset(&(port->port_status),
2355                                         RTE_PORT_HANDLING,
2356                                         RTE_PORT_STOPPED) == 0)
2357                         printf("Port %d can not be set back "
2358                                         "to stopped\n", pi);
2359                 printf("Fail to configure port %d hairpin "
2360                                 "queues\n", pi);
2361                 /* try to reconfigure queues next time */
2362                 port->need_reconfig_queues = 1;
2363                 return -1;
2364         }
2365         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2366                 hairpin_conf.peers[0].port = pi;
2367                 hairpin_conf.peers[0].queue = i + nb_txq;
2368                 diag = rte_eth_rx_hairpin_queue_setup
2369                         (pi, qi, nb_rxd, &hairpin_conf);
2370                 i++;
2371                 if (diag == 0)
2372                         continue;
2373
2374                 /* Fail to setup rx queue, return */
2375                 if (rte_atomic16_cmpset(&(port->port_status),
2376                                         RTE_PORT_HANDLING,
2377                                         RTE_PORT_STOPPED) == 0)
2378                         printf("Port %d can not be set back "
2379                                         "to stopped\n", pi);
2380                 printf("Fail to configure port %d hairpin "
2381                                 "queues\n", pi);
2382                 /* try to reconfigure queues next time */
2383                 port->need_reconfig_queues = 1;
2384                 return -1;
2385         }
2386         return 0;
2387 }
2388
2389 int
2390 start_port(portid_t pid)
2391 {
2392         int diag, need_check_link_status = -1;
2393         portid_t pi;
2394         queueid_t qi;
2395         struct rte_port *port;
2396         struct rte_ether_addr mac_addr;
2397         struct rte_eth_hairpin_cap cap;
2398
2399         if (port_id_is_invalid(pid, ENABLED_WARN))
2400                 return 0;
2401
2402         if(dcb_config)
2403                 dcb_test = 1;
2404         RTE_ETH_FOREACH_DEV(pi) {
2405                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2406                         continue;
2407
2408                 need_check_link_status = 0;
2409                 port = &ports[pi];
2410                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2411                                                  RTE_PORT_HANDLING) == 0) {
2412                         printf("Port %d is now not stopped\n", pi);
2413                         continue;
2414                 }
2415
2416                 if (port->need_reconfig > 0) {
2417                         port->need_reconfig = 0;
2418
2419                         if (flow_isolate_all) {
2420                                 int ret = port_flow_isolate(pi, 1);
2421                                 if (ret) {
2422                                         printf("Failed to apply isolated"
2423                                                " mode on port %d\n", pi);
2424                                         return -1;
2425                                 }
2426                         }
2427                         configure_rxtx_dump_callbacks(0);
2428                         printf("Configuring Port %d (socket %u)\n", pi,
2429                                         port->socket_id);
2430                         if (nb_hairpinq > 0 &&
2431                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2432                                 printf("Port %d doesn't support hairpin "
2433                                        "queues\n", pi);
2434                                 return -1;
2435                         }
2436                         /* configure port */
2437                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2438                                                      nb_txq + nb_hairpinq,
2439                                                      &(port->dev_conf));
2440                         if (diag != 0) {
2441                                 if (rte_atomic16_cmpset(&(port->port_status),
2442                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2443                                         printf("Port %d can not be set back "
2444                                                         "to stopped\n", pi);
2445                                 printf("Fail to configure port %d\n", pi);
2446                                 /* try to reconfigure port next time */
2447                                 port->need_reconfig = 1;
2448                                 return -1;
2449                         }
2450                 }
2451                 if (port->need_reconfig_queues > 0) {
2452                         port->need_reconfig_queues = 0;
2453                         /* setup tx queues */
2454                         for (qi = 0; qi < nb_txq; qi++) {
2455                                 if ((numa_support) &&
2456                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2457                                         diag = rte_eth_tx_queue_setup(pi, qi,
2458                                                 port->nb_tx_desc[qi],
2459                                                 txring_numa[pi],
2460                                                 &(port->tx_conf[qi]));
2461                                 else
2462                                         diag = rte_eth_tx_queue_setup(pi, qi,
2463                                                 port->nb_tx_desc[qi],
2464                                                 port->socket_id,
2465                                                 &(port->tx_conf[qi]));
2466
2467                                 if (diag == 0)
2468                                         continue;
2469
2470                                 /* Fail to setup tx queue, return */
2471                                 if (rte_atomic16_cmpset(&(port->port_status),
2472                                                         RTE_PORT_HANDLING,
2473                                                         RTE_PORT_STOPPED) == 0)
2474                                         printf("Port %d can not be set back "
2475                                                         "to stopped\n", pi);
2476                                 printf("Fail to configure port %d tx queues\n",
2477                                        pi);
2478                                 /* try to reconfigure queues next time */
2479                                 port->need_reconfig_queues = 1;
2480                                 return -1;
2481                         }
2482                         for (qi = 0; qi < nb_rxq; qi++) {
2483                                 /* setup rx queues */
2484                                 if ((numa_support) &&
2485                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2486                                         struct rte_mempool * mp =
2487                                                 mbuf_pool_find(rxring_numa[pi]);
2488                                         if (mp == NULL) {
2489                                                 printf("Failed to setup RX queue:"
2490                                                         "No mempool allocation"
2491                                                         " on the socket %d\n",
2492                                                         rxring_numa[pi]);
2493                                                 return -1;
2494                                         }
2495
2496                                         diag = rte_eth_rx_queue_setup(pi, qi,
2497                                              port->nb_rx_desc[qi],
2498                                              rxring_numa[pi],
2499                                              &(port->rx_conf[qi]),
2500                                              mp);
2501                                 } else {
2502                                         struct rte_mempool *mp =
2503                                                 mbuf_pool_find(port->socket_id);
2504                                         if (mp == NULL) {
2505                                                 printf("Failed to setup RX queue:"
2506                                                         "No mempool allocation"
2507                                                         " on the socket %d\n",
2508                                                         port->socket_id);
2509                                                 return -1;
2510                                         }
2511                                         diag = rte_eth_rx_queue_setup(pi, qi,
2512                                              port->nb_rx_desc[qi],
2513                                              port->socket_id,
2514                                              &(port->rx_conf[qi]),
2515                                              mp);
2516                                 }
2517                                 if (diag == 0)
2518                                         continue;
2519
2520                                 /* Fail to setup rx queue, return */
2521                                 if (rte_atomic16_cmpset(&(port->port_status),
2522                                                         RTE_PORT_HANDLING,
2523                                                         RTE_PORT_STOPPED) == 0)
2524                                         printf("Port %d can not be set back "
2525                                                         "to stopped\n", pi);
2526                                 printf("Fail to configure port %d rx queues\n",
2527                                        pi);
2528                                 /* try to reconfigure queues next time */
2529                                 port->need_reconfig_queues = 1;
2530                                 return -1;
2531                         }
2532                         /* setup hairpin queues */
2533                         if (setup_hairpin_queues(pi) != 0)
2534                                 return -1;
2535                 }
2536                 configure_rxtx_dump_callbacks(verbose_level);
2537                 if (clear_ptypes) {
2538                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2539                                         NULL, 0);
2540                         if (diag < 0)
2541                                 printf(
2542                                 "Port %d: Failed to disable Ptype parsing\n",
2543                                 pi);
2544                 }
2545
2546                 /* start port */
2547                 if (rte_eth_dev_start(pi) < 0) {
2548                         printf("Fail to start port %d\n", pi);
2549
2550                         /* Fail to setup rx queue, return */
2551                         if (rte_atomic16_cmpset(&(port->port_status),
2552                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2553                                 printf("Port %d can not be set back to "
2554                                                         "stopped\n", pi);
2555                         continue;
2556                 }
2557
2558                 if (rte_atomic16_cmpset(&(port->port_status),
2559                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2560                         printf("Port %d can not be set into started\n", pi);
2561
2562                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2563                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2564                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2565                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2566                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2567
2568                 /* at least one port started, need checking link status */
2569                 need_check_link_status = 1;
2570         }
2571
2572         if (need_check_link_status == 1 && !no_link_check)
2573                 check_all_ports_link_status(RTE_PORT_ALL);
2574         else if (need_check_link_status == 0)
2575                 printf("Please stop the ports first\n");
2576
2577         printf("Done\n");
2578         return 0;
2579 }
2580
2581 void
2582 stop_port(portid_t pid)
2583 {
2584         portid_t pi;
2585         struct rte_port *port;
2586         int need_check_link_status = 0;
2587
2588         if (dcb_test) {
2589                 dcb_test = 0;
2590                 dcb_config = 0;
2591         }
2592
2593         if (port_id_is_invalid(pid, ENABLED_WARN))
2594                 return;
2595
2596         printf("Stopping ports...\n");
2597
2598         RTE_ETH_FOREACH_DEV(pi) {
2599                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2600                         continue;
2601
2602                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2603                         printf("Please remove port %d from forwarding configuration.\n", pi);
2604                         continue;
2605                 }
2606
2607                 if (port_is_bonding_slave(pi)) {
2608                         printf("Please remove port %d from bonded device.\n", pi);
2609                         continue;
2610                 }
2611
2612                 port = &ports[pi];
2613                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2614                                                 RTE_PORT_HANDLING) == 0)
2615                         continue;
2616
2617                 rte_eth_dev_stop(pi);
2618
2619                 if (rte_atomic16_cmpset(&(port->port_status),
2620                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2621                         printf("Port %d can not be set into stopped\n", pi);
2622                 need_check_link_status = 1;
2623         }
2624         if (need_check_link_status && !no_link_check)
2625                 check_all_ports_link_status(RTE_PORT_ALL);
2626
2627         printf("Done\n");
2628 }
2629
2630 static void
2631 remove_invalid_ports_in(portid_t *array, portid_t *total)
2632 {
2633         portid_t i;
2634         portid_t new_total = 0;
2635
2636         for (i = 0; i < *total; i++)
2637                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2638                         array[new_total] = array[i];
2639                         new_total++;
2640                 }
2641         *total = new_total;
2642 }
2643
2644 static void
2645 remove_invalid_ports(void)
2646 {
2647         remove_invalid_ports_in(ports_ids, &nb_ports);
2648         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2649         nb_cfg_ports = nb_fwd_ports;
2650 }
2651
2652 void
2653 close_port(portid_t pid)
2654 {
2655         portid_t pi;
2656         struct rte_port *port;
2657
2658         if (port_id_is_invalid(pid, ENABLED_WARN))
2659                 return;
2660
2661         printf("Closing ports...\n");
2662
2663         RTE_ETH_FOREACH_DEV(pi) {
2664                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2665                         continue;
2666
2667                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2668                         printf("Please remove port %d from forwarding configuration.\n", pi);
2669                         continue;
2670                 }
2671
2672                 if (port_is_bonding_slave(pi)) {
2673                         printf("Please remove port %d from bonded device.\n", pi);
2674                         continue;
2675                 }
2676
2677                 port = &ports[pi];
2678                 if (rte_atomic16_cmpset(&(port->port_status),
2679                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2680                         printf("Port %d is already closed\n", pi);
2681                         continue;
2682                 }
2683
2684                 if (rte_atomic16_cmpset(&(port->port_status),
2685                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2686                         printf("Port %d is now not stopped\n", pi);
2687                         continue;
2688                 }
2689
2690                 if (port->flow_list)
2691                         port_flow_flush(pi);
2692                 rte_eth_dev_close(pi);
2693
2694                 remove_invalid_ports();
2695
2696                 if (rte_atomic16_cmpset(&(port->port_status),
2697                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2698                         printf("Port %d cannot be set to closed\n", pi);
2699         }
2700
2701         printf("Done\n");
2702 }
2703
2704 void
2705 reset_port(portid_t pid)
2706 {
2707         int diag;
2708         portid_t pi;
2709         struct rte_port *port;
2710
2711         if (port_id_is_invalid(pid, ENABLED_WARN))
2712                 return;
2713
2714         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2715                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2716                 printf("Can not reset port(s), please stop port(s) first.\n");
2717                 return;
2718         }
2719
2720         printf("Resetting ports...\n");
2721
2722         RTE_ETH_FOREACH_DEV(pi) {
2723                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2724                         continue;
2725
2726                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2727                         printf("Please remove port %d from forwarding "
2728                                "configuration.\n", pi);
2729                         continue;
2730                 }
2731
2732                 if (port_is_bonding_slave(pi)) {
2733                         printf("Please remove port %d from bonded device.\n",
2734                                pi);
2735                         continue;
2736                 }
2737
2738                 diag = rte_eth_dev_reset(pi);
2739                 if (diag == 0) {
2740                         port = &ports[pi];
2741                         port->need_reconfig = 1;
2742                         port->need_reconfig_queues = 1;
2743                 } else {
2744                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2745                 }
2746         }
2747
2748         printf("Done\n");
2749 }
2750
2751 void
2752 attach_port(char *identifier)
2753 {
2754         portid_t pi;
2755         struct rte_dev_iterator iterator;
2756
2757         printf("Attaching a new port...\n");
2758
2759         if (identifier == NULL) {
2760                 printf("Invalid parameters are specified\n");
2761                 return;
2762         }
2763
2764         if (rte_dev_probe(identifier) < 0) {
2765                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2766                 return;
2767         }
2768
2769         /* first attach mode: event */
2770         if (setup_on_probe_event) {
2771                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2772                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2773                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2774                                         ports[pi].need_setup != 0)
2775                                 setup_attached_port(pi);
2776                 return;
2777         }
2778
2779         /* second attach mode: iterator */
2780         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2781                 /* setup ports matching the devargs used for probing */
2782                 if (port_is_forwarding(pi))
2783                         continue; /* port was already attached before */
2784                 setup_attached_port(pi);
2785         }
2786 }
2787
2788 static void
2789 setup_attached_port(portid_t pi)
2790 {
2791         unsigned int socket_id;
2792         int ret;
2793
2794         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2795         /* if socket_id is invalid, set to the first available socket. */
2796         if (check_socket_id(socket_id) < 0)
2797                 socket_id = socket_ids[0];
2798         reconfig(pi, socket_id);
2799         ret = rte_eth_promiscuous_enable(pi);
2800         if (ret != 0)
2801                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2802                         pi, rte_strerror(-ret));
2803
2804         ports_ids[nb_ports++] = pi;
2805         fwd_ports_ids[nb_fwd_ports++] = pi;
2806         nb_cfg_ports = nb_fwd_ports;
2807         ports[pi].need_setup = 0;
2808         ports[pi].port_status = RTE_PORT_STOPPED;
2809
2810         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2811         printf("Done\n");
2812 }
2813
2814 static void
2815 detach_device(struct rte_device *dev)
2816 {
2817         portid_t sibling;
2818
2819         if (dev == NULL) {
2820                 printf("Device already removed\n");
2821                 return;
2822         }
2823
2824         printf("Removing a device...\n");
2825
2826         if (rte_dev_remove(dev) < 0) {
2827                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2828                 return;
2829         }
2830         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2831                 /* reset mapping between old ports and removed device */
2832                 rte_eth_devices[sibling].device = NULL;
2833                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2834                         /* sibling ports are forced to be closed */
2835                         ports[sibling].port_status = RTE_PORT_CLOSED;
2836                         printf("Port %u is closed\n", sibling);
2837                 }
2838         }
2839
2840         remove_invalid_ports();
2841
2842         printf("Device is detached\n");
2843         printf("Now total ports is %d\n", nb_ports);
2844         printf("Done\n");
2845         return;
2846 }
2847
2848 void
2849 detach_port_device(portid_t port_id)
2850 {
2851         if (port_id_is_invalid(port_id, ENABLED_WARN))
2852                 return;
2853
2854         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2855                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2856                         printf("Port not stopped\n");
2857                         return;
2858                 }
2859                 printf("Port was not closed\n");
2860                 if (ports[port_id].flow_list)
2861                         port_flow_flush(port_id);
2862         }
2863
2864         detach_device(rte_eth_devices[port_id].device);
2865 }
2866
2867 void
2868 detach_devargs(char *identifier)
2869 {
2870         struct rte_dev_iterator iterator;
2871         struct rte_devargs da;
2872         portid_t port_id;
2873
2874         printf("Removing a device...\n");
2875
2876         memset(&da, 0, sizeof(da));
2877         if (rte_devargs_parsef(&da, "%s", identifier)) {
2878                 printf("cannot parse identifier\n");
2879                 if (da.args)
2880                         free(da.args);
2881                 return;
2882         }
2883
2884         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2885                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2886                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2887                                 printf("Port %u not stopped\n", port_id);
2888                                 rte_eth_iterator_cleanup(&iterator);
2889                                 return;
2890                         }
2891
2892                         /* sibling ports are forced to be closed */
2893                         if (ports[port_id].flow_list)
2894                                 port_flow_flush(port_id);
2895                         ports[port_id].port_status = RTE_PORT_CLOSED;
2896                         printf("Port %u is now closed\n", port_id);
2897                 }
2898         }
2899
2900         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2901                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2902                             da.name, da.bus->name);
2903                 return;
2904         }
2905
2906         remove_invalid_ports();
2907
2908         printf("Device %s is detached\n", identifier);
2909         printf("Now total ports is %d\n", nb_ports);
2910         printf("Done\n");
2911 }
2912
2913 void
2914 pmd_test_exit(void)
2915 {
2916         portid_t pt_id;
2917         int ret;
2918         int i;
2919
2920         if (test_done == 0)
2921                 stop_packet_forwarding();
2922
2923         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2924                 if (mempools[i]) {
2925                         if (mp_alloc_type == MP_ALLOC_ANON)
2926                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2927                                                      NULL);
2928                 }
2929         }
2930         if (ports != NULL) {
2931                 no_link_check = 1;
2932                 RTE_ETH_FOREACH_DEV(pt_id) {
2933                         printf("\nStopping port %d...\n", pt_id);
2934                         fflush(stdout);
2935                         stop_port(pt_id);
2936                 }
2937                 RTE_ETH_FOREACH_DEV(pt_id) {
2938                         printf("\nShutting down port %d...\n", pt_id);
2939                         fflush(stdout);
2940                         close_port(pt_id);
2941                 }
2942         }
2943
2944         if (hot_plug) {
2945                 ret = rte_dev_event_monitor_stop();
2946                 if (ret) {
2947                         RTE_LOG(ERR, EAL,
2948                                 "fail to stop device event monitor.");
2949                         return;
2950                 }
2951
2952                 ret = rte_dev_event_callback_unregister(NULL,
2953                         dev_event_callback, NULL);
2954                 if (ret < 0) {
2955                         RTE_LOG(ERR, EAL,
2956                                 "fail to unregister device event callback.\n");
2957                         return;
2958                 }
2959
2960                 ret = rte_dev_hotplug_handle_disable();
2961                 if (ret) {
2962                         RTE_LOG(ERR, EAL,
2963                                 "fail to disable hotplug handling.\n");
2964                         return;
2965                 }
2966         }
2967         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2968                 if (mempools[i])
2969                         rte_mempool_free(mempools[i]);
2970         }
2971
2972         printf("\nBye...\n");
2973 }
2974
2975 typedef void (*cmd_func_t)(void);
2976 struct pmd_test_command {
2977         const char *cmd_name;
2978         cmd_func_t cmd_func;
2979 };
2980
2981 /* Check the link status of all ports in up to 9s, and print them finally */
2982 static void
2983 check_all_ports_link_status(uint32_t port_mask)
2984 {
2985 #define CHECK_INTERVAL 100 /* 100ms */
2986 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2987         portid_t portid;
2988         uint8_t count, all_ports_up, print_flag = 0;
2989         struct rte_eth_link link;
2990         int ret;
2991
2992         printf("Checking link statuses...\n");
2993         fflush(stdout);
2994         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2995                 all_ports_up = 1;
2996                 RTE_ETH_FOREACH_DEV(portid) {
2997                         if ((port_mask & (1 << portid)) == 0)
2998                                 continue;
2999                         memset(&link, 0, sizeof(link));
3000                         ret = rte_eth_link_get_nowait(portid, &link);
3001                         if (ret < 0) {
3002                                 all_ports_up = 0;
3003                                 if (print_flag == 1)
3004                                         printf("Port %u link get failed: %s\n",
3005                                                 portid, rte_strerror(-ret));
3006                                 continue;
3007                         }
3008                         /* print link status if flag set */
3009                         if (print_flag == 1) {
3010                                 if (link.link_status)
3011                                         printf(
3012                                         "Port%d Link Up. speed %u Mbps- %s\n",
3013                                         portid, link.link_speed,
3014                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3015                                         ("full-duplex") : ("half-duplex"));
3016                                 else
3017                                         printf("Port %d Link Down\n", portid);
3018                                 continue;
3019                         }
3020                         /* clear all_ports_up flag if any link down */
3021                         if (link.link_status == ETH_LINK_DOWN) {
3022                                 all_ports_up = 0;
3023                                 break;
3024                         }
3025                 }
3026                 /* after finally printing all link status, get out */
3027                 if (print_flag == 1)
3028                         break;
3029
3030                 if (all_ports_up == 0) {
3031                         fflush(stdout);
3032                         rte_delay_ms(CHECK_INTERVAL);
3033                 }
3034
3035                 /* set the print_flag if all ports up or timeout */
3036                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3037                         print_flag = 1;
3038                 }
3039
3040                 if (lsc_interrupt)
3041                         break;
3042         }
3043 }
3044
3045 /*
3046  * This callback is for remove a port for a device. It has limitation because
3047  * it is not for multiple port removal for a device.
3048  * TODO: the device detach invoke will plan to be removed from user side to
3049  * eal. And convert all PMDs to free port resources on ether device closing.
3050  */
3051 static void
3052 rmv_port_callback(void *arg)
3053 {
3054         int need_to_start = 0;
3055         int org_no_link_check = no_link_check;
3056         portid_t port_id = (intptr_t)arg;
3057         struct rte_device *dev;
3058
3059         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3060
3061         if (!test_done && port_is_forwarding(port_id)) {
3062                 need_to_start = 1;
3063                 stop_packet_forwarding();
3064         }
3065         no_link_check = 1;
3066         stop_port(port_id);
3067         no_link_check = org_no_link_check;
3068
3069         /* Save rte_device pointer before closing ethdev port */
3070         dev = rte_eth_devices[port_id].device;
3071         close_port(port_id);
3072         detach_device(dev); /* might be already removed or have more ports */
3073
3074         if (need_to_start)
3075                 start_packet_forwarding(0);
3076 }
3077
3078 /* This function is used by the interrupt thread */
3079 static int
3080 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3081                   void *ret_param)
3082 {
3083         RTE_SET_USED(param);
3084         RTE_SET_USED(ret_param);
3085
3086         if (type >= RTE_ETH_EVENT_MAX) {
3087                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3088                         port_id, __func__, type);
3089                 fflush(stderr);
3090         } else if (event_print_mask & (UINT32_C(1) << type)) {
3091                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3092                         eth_event_desc[type]);
3093                 fflush(stdout);
3094         }
3095
3096         switch (type) {
3097         case RTE_ETH_EVENT_NEW:
3098                 ports[port_id].need_setup = 1;
3099                 ports[port_id].port_status = RTE_PORT_HANDLING;
3100                 break;
3101         case RTE_ETH_EVENT_INTR_RMV:
3102                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3103                         break;
3104                 if (rte_eal_alarm_set(100000,
3105                                 rmv_port_callback, (void *)(intptr_t)port_id))
3106                         fprintf(stderr, "Could not set up deferred device removal\n");
3107                 break;
3108         default:
3109                 break;
3110         }
3111         return 0;
3112 }
3113
3114 static int
3115 register_eth_event_callback(void)
3116 {
3117         int ret;
3118         enum rte_eth_event_type event;
3119
3120         for (event = RTE_ETH_EVENT_UNKNOWN;
3121                         event < RTE_ETH_EVENT_MAX; event++) {
3122                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3123                                 event,
3124                                 eth_event_callback,
3125                                 NULL);
3126                 if (ret != 0) {
3127                         TESTPMD_LOG(ERR, "Failed to register callback for "
3128                                         "%s event\n", eth_event_desc[event]);
3129                         return -1;
3130                 }
3131         }
3132
3133         return 0;
3134 }
3135
3136 /* This function is used by the interrupt thread */
3137 static void
3138 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3139                              __rte_unused void *arg)
3140 {
3141         uint16_t port_id;
3142         int ret;
3143
3144         if (type >= RTE_DEV_EVENT_MAX) {
3145                 fprintf(stderr, "%s called upon invalid event %d\n",
3146                         __func__, type);
3147                 fflush(stderr);
3148         }
3149
3150         switch (type) {
3151         case RTE_DEV_EVENT_REMOVE:
3152                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3153                         device_name);
3154                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3155                 if (ret) {
3156                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3157                                 device_name);
3158                         return;
3159                 }
3160                 /*
3161                  * Because the user's callback is invoked in eal interrupt
3162                  * callback, the interrupt callback need to be finished before
3163                  * it can be unregistered when detaching device. So finish
3164                  * callback soon and use a deferred removal to detach device
3165                  * is need. It is a workaround, once the device detaching be
3166                  * moved into the eal in the future, the deferred removal could
3167                  * be deleted.
3168                  */
3169                 if (rte_eal_alarm_set(100000,
3170                                 rmv_port_callback, (void *)(intptr_t)port_id))
3171                         RTE_LOG(ERR, EAL,
3172                                 "Could not set up deferred device removal\n");
3173                 break;
3174         case RTE_DEV_EVENT_ADD:
3175                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3176                         device_name);
3177                 /* TODO: After finish kernel driver binding,
3178                  * begin to attach port.
3179                  */
3180                 break;
3181         default:
3182                 break;
3183         }
3184 }
3185
3186 static int
3187 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3188 {
3189         uint16_t i;
3190         int diag;
3191         uint8_t mapping_found = 0;
3192
3193         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3194                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3195                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3196                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3197                                         tx_queue_stats_mappings[i].queue_id,
3198                                         tx_queue_stats_mappings[i].stats_counter_id);
3199                         if (diag != 0)
3200                                 return diag;
3201                         mapping_found = 1;
3202                 }
3203         }
3204         if (mapping_found)
3205                 port->tx_queue_stats_mapping_enabled = 1;
3206         return 0;
3207 }
3208
3209 static int
3210 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3211 {
3212         uint16_t i;
3213         int diag;
3214         uint8_t mapping_found = 0;
3215
3216         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3217                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3218                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3219                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3220                                         rx_queue_stats_mappings[i].queue_id,
3221                                         rx_queue_stats_mappings[i].stats_counter_id);
3222                         if (diag != 0)
3223                                 return diag;
3224                         mapping_found = 1;
3225                 }
3226         }
3227         if (mapping_found)
3228                 port->rx_queue_stats_mapping_enabled = 1;
3229         return 0;
3230 }
3231
3232 static void
3233 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3234 {
3235         int diag = 0;
3236
3237         diag = set_tx_queue_stats_mapping_registers(pi, port);
3238         if (diag != 0) {
3239                 if (diag == -ENOTSUP) {
3240                         port->tx_queue_stats_mapping_enabled = 0;
3241                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3242                 }
3243                 else
3244                         rte_exit(EXIT_FAILURE,
3245                                         "set_tx_queue_stats_mapping_registers "
3246                                         "failed for port id=%d diag=%d\n",
3247                                         pi, diag);
3248         }
3249
3250         diag = set_rx_queue_stats_mapping_registers(pi, port);
3251         if (diag != 0) {
3252                 if (diag == -ENOTSUP) {
3253                         port->rx_queue_stats_mapping_enabled = 0;
3254                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3255                 }
3256                 else
3257                         rte_exit(EXIT_FAILURE,
3258                                         "set_rx_queue_stats_mapping_registers "
3259                                         "failed for port id=%d diag=%d\n",
3260                                         pi, diag);
3261         }
3262 }
3263
3264 static void
3265 rxtx_port_config(struct rte_port *port)
3266 {
3267         uint16_t qid;
3268         uint64_t offloads;
3269
3270         for (qid = 0; qid < nb_rxq; qid++) {
3271                 offloads = port->rx_conf[qid].offloads;
3272                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3273                 if (offloads != 0)
3274                         port->rx_conf[qid].offloads = offloads;
3275
3276                 /* Check if any Rx parameters have been passed */
3277                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3278                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3279
3280                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3281                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3282
3283                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3284                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3285
3286                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3287                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3288
3289                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3290                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3291
3292                 port->nb_rx_desc[qid] = nb_rxd;
3293         }
3294
3295         for (qid = 0; qid < nb_txq; qid++) {
3296                 offloads = port->tx_conf[qid].offloads;
3297                 port->tx_conf[qid] = port->dev_info.default_txconf;
3298                 if (offloads != 0)
3299                         port->tx_conf[qid].offloads = offloads;
3300
3301                 /* Check if any Tx parameters have been passed */
3302                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3303                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3304
3305                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3306                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3307
3308                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3309                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3310
3311                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3312                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3313
3314                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3315                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3316
3317                 port->nb_tx_desc[qid] = nb_txd;
3318         }
3319 }
3320
3321 void
3322 init_port_config(void)
3323 {
3324         portid_t pid;
3325         struct rte_port *port;
3326         int ret;
3327
3328         RTE_ETH_FOREACH_DEV(pid) {
3329                 port = &ports[pid];
3330                 port->dev_conf.fdir_conf = fdir_conf;
3331
3332                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3333                 if (ret != 0)
3334                         return;
3335
3336                 if (nb_rxq > 1) {
3337                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3338                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3339                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3340                 } else {
3341                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3342                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3343                 }
3344
3345                 if (port->dcb_flag == 0) {
3346                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3347                                 port->dev_conf.rxmode.mq_mode =
3348                                         (enum rte_eth_rx_mq_mode)
3349                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3350                         else
3351                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3352                 }
3353
3354                 rxtx_port_config(port);
3355
3356                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3357                 if (ret != 0)
3358                         return;
3359
3360                 map_port_queue_stats_mapping_registers(pid, port);
3361 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3362                 rte_pmd_ixgbe_bypass_init(pid);
3363 #endif
3364
3365                 if (lsc_interrupt &&
3366                     (rte_eth_devices[pid].data->dev_flags &
3367                      RTE_ETH_DEV_INTR_LSC))
3368                         port->dev_conf.intr_conf.lsc = 1;
3369                 if (rmv_interrupt &&
3370                     (rte_eth_devices[pid].data->dev_flags &
3371                      RTE_ETH_DEV_INTR_RMV))
3372                         port->dev_conf.intr_conf.rmv = 1;
3373         }
3374 }
3375
3376 void set_port_slave_flag(portid_t slave_pid)
3377 {
3378         struct rte_port *port;
3379
3380         port = &ports[slave_pid];
3381         port->slave_flag = 1;
3382 }
3383
3384 void clear_port_slave_flag(portid_t slave_pid)
3385 {
3386         struct rte_port *port;
3387
3388         port = &ports[slave_pid];
3389         port->slave_flag = 0;
3390 }
3391
3392 uint8_t port_is_bonding_slave(portid_t slave_pid)
3393 {
3394         struct rte_port *port;
3395
3396         port = &ports[slave_pid];
3397         if ((rte_eth_devices[slave_pid].data->dev_flags &
3398             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3399                 return 1;
3400         return 0;
3401 }
3402
3403 const uint16_t vlan_tags[] = {
3404                 0,  1,  2,  3,  4,  5,  6,  7,
3405                 8,  9, 10, 11,  12, 13, 14, 15,
3406                 16, 17, 18, 19, 20, 21, 22, 23,
3407                 24, 25, 26, 27, 28, 29, 30, 31
3408 };
3409
3410 static  int
3411 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3412                  enum dcb_mode_enable dcb_mode,
3413                  enum rte_eth_nb_tcs num_tcs,
3414                  uint8_t pfc_en)
3415 {
3416         uint8_t i;
3417         int32_t rc;
3418         struct rte_eth_rss_conf rss_conf;
3419
3420         /*
3421          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3422          * given above, and the number of traffic classes available for use.
3423          */
3424         if (dcb_mode == DCB_VT_ENABLED) {
3425                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3426                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3427                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3428                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3429
3430                 /* VMDQ+DCB RX and TX configurations */
3431                 vmdq_rx_conf->enable_default_pool = 0;
3432                 vmdq_rx_conf->default_pool = 0;
3433                 vmdq_rx_conf->nb_queue_pools =
3434                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3435                 vmdq_tx_conf->nb_queue_pools =
3436                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3437
3438                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3439                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3440                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3441                         vmdq_rx_conf->pool_map[i].pools =
3442                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3443                 }
3444                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3445                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3446                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3447                 }
3448
3449                 /* set DCB mode of RX and TX of multiple queues */
3450                 eth_conf->rxmode.mq_mode =
3451                                 (enum rte_eth_rx_mq_mode)
3452                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3453                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3454         } else {
3455                 struct rte_eth_dcb_rx_conf *rx_conf =
3456                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3457                 struct rte_eth_dcb_tx_conf *tx_conf =
3458                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3459
3460                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3461                 if (rc != 0)
3462                         return rc;
3463
3464                 rx_conf->nb_tcs = num_tcs;
3465                 tx_conf->nb_tcs = num_tcs;
3466
3467                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3468                         rx_conf->dcb_tc[i] = i % num_tcs;
3469                         tx_conf->dcb_tc[i] = i % num_tcs;
3470                 }
3471
3472                 eth_conf->rxmode.mq_mode =
3473                                 (enum rte_eth_rx_mq_mode)
3474                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3475                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3476                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3477         }
3478
3479         if (pfc_en)
3480                 eth_conf->dcb_capability_en =
3481                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3482         else
3483                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3484
3485         return 0;
3486 }
3487
3488 int
3489 init_port_dcb_config(portid_t pid,
3490                      enum dcb_mode_enable dcb_mode,
3491                      enum rte_eth_nb_tcs num_tcs,
3492                      uint8_t pfc_en)
3493 {
3494         struct rte_eth_conf port_conf;
3495         struct rte_port *rte_port;
3496         int retval;
3497         uint16_t i;
3498
3499         rte_port = &ports[pid];
3500
3501         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3502         /* Enter DCB configuration status */
3503         dcb_config = 1;
3504
3505         port_conf.rxmode = rte_port->dev_conf.rxmode;
3506         port_conf.txmode = rte_port->dev_conf.txmode;
3507
3508         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3509         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3510         if (retval < 0)
3511                 return retval;
3512         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3513
3514         /* re-configure the device . */
3515         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3516         if (retval < 0)
3517                 return retval;
3518
3519         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3520         if (retval != 0)
3521                 return retval;
3522
3523         /* If dev_info.vmdq_pool_base is greater than 0,
3524          * the queue id of vmdq pools is started after pf queues.
3525          */
3526         if (dcb_mode == DCB_VT_ENABLED &&
3527             rte_port->dev_info.vmdq_pool_base > 0) {
3528                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3529                         " for port %d.", pid);
3530                 return -1;
3531         }
3532
3533         /* Assume the ports in testpmd have the same dcb capability
3534          * and has the same number of rxq and txq in dcb mode
3535          */
3536         if (dcb_mode == DCB_VT_ENABLED) {
3537                 if (rte_port->dev_info.max_vfs > 0) {
3538                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3539                         nb_txq = rte_port->dev_info.nb_tx_queues;
3540                 } else {
3541                         nb_rxq = rte_port->dev_info.max_rx_queues;
3542                         nb_txq = rte_port->dev_info.max_tx_queues;
3543                 }
3544         } else {
3545                 /*if vt is disabled, use all pf queues */
3546                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3547                         nb_rxq = rte_port->dev_info.max_rx_queues;
3548                         nb_txq = rte_port->dev_info.max_tx_queues;
3549                 } else {
3550                         nb_rxq = (queueid_t)num_tcs;
3551                         nb_txq = (queueid_t)num_tcs;
3552
3553                 }
3554         }
3555         rx_free_thresh = 64;
3556
3557         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3558
3559         rxtx_port_config(rte_port);
3560         /* VLAN filter */
3561         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3562         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3563                 rx_vft_set(pid, vlan_tags[i], 1);
3564
3565         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3566         if (retval != 0)
3567                 return retval;
3568
3569         map_port_queue_stats_mapping_registers(pid, rte_port);
3570
3571         rte_port->dcb_flag = 1;
3572
3573         return 0;
3574 }
3575
3576 static void
3577 init_port(void)
3578 {
3579         /* Configuration of Ethernet ports. */
3580         ports = rte_zmalloc("testpmd: ports",
3581                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3582                             RTE_CACHE_LINE_SIZE);
3583         if (ports == NULL) {
3584                 rte_exit(EXIT_FAILURE,
3585                                 "rte_zmalloc(%d struct rte_port) failed\n",
3586                                 RTE_MAX_ETHPORTS);
3587         }
3588
3589         /* Initialize ports NUMA structures */
3590         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3591         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3592         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3593 }
3594
3595 static void
3596 force_quit(void)
3597 {
3598         pmd_test_exit();
3599         prompt_exit();
3600 }
3601
3602 static void
3603 print_stats(void)
3604 {
3605         uint8_t i;
3606         const char clr[] = { 27, '[', '2', 'J', '\0' };
3607         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3608
3609         /* Clear screen and move to top left */
3610         printf("%s%s", clr, top_left);
3611
3612         printf("\nPort statistics ====================================");
3613         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3614                 nic_stats_display(fwd_ports_ids[i]);
3615
3616         fflush(stdout);
3617 }
3618
3619 static void
3620 signal_handler(int signum)
3621 {
3622         if (signum == SIGINT || signum == SIGTERM) {
3623                 printf("\nSignal %d received, preparing to exit...\n",
3624                                 signum);
3625 #ifdef RTE_LIBRTE_PDUMP
3626                 /* uninitialize packet capture framework */
3627                 rte_pdump_uninit();
3628 #endif
3629 #ifdef RTE_LIBRTE_LATENCY_STATS
3630                 if (latencystats_enabled != 0)
3631                         rte_latencystats_uninit();
3632 #endif
3633                 force_quit();
3634                 /* Set flag to indicate the force termination. */
3635                 f_quit = 1;
3636                 /* exit with the expected status */
3637                 signal(signum, SIG_DFL);
3638                 kill(getpid(), signum);
3639         }
3640 }
3641
3642 int
3643 main(int argc, char** argv)
3644 {
3645         int diag;
3646         portid_t port_id;
3647         uint16_t count;
3648         int ret;
3649
3650         signal(SIGINT, signal_handler);
3651         signal(SIGTERM, signal_handler);
3652
3653         testpmd_logtype = rte_log_register("testpmd");
3654         if (testpmd_logtype < 0)
3655                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3656         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3657
3658         diag = rte_eal_init(argc, argv);
3659         if (diag < 0)
3660                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3661                          rte_strerror(rte_errno));
3662
3663         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3664                 rte_exit(EXIT_FAILURE,
3665                          "Secondary process type not supported.\n");
3666
3667         ret = register_eth_event_callback();
3668         if (ret != 0)
3669                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3670
3671 #ifdef RTE_LIBRTE_PDUMP
3672         /* initialize packet capture framework */
3673         rte_pdump_init();
3674 #endif
3675
3676         count = 0;
3677         RTE_ETH_FOREACH_DEV(port_id) {
3678                 ports_ids[count] = port_id;
3679                 count++;
3680         }
3681         nb_ports = (portid_t) count;
3682         if (nb_ports == 0)
3683                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3684
3685         /* allocate port structures, and init them */
3686         init_port();
3687
3688         set_def_fwd_config();
3689         if (nb_lcores == 0)
3690                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3691                          "Check the core mask argument\n");
3692
3693         /* Bitrate/latency stats disabled by default */
3694 #ifdef RTE_LIBRTE_BITRATE
3695         bitrate_enabled = 0;
3696 #endif
3697 #ifdef RTE_LIBRTE_LATENCY_STATS
3698         latencystats_enabled = 0;
3699 #endif
3700
3701         /* on FreeBSD, mlockall() is disabled by default */
3702 #ifdef RTE_EXEC_ENV_FREEBSD
3703         do_mlockall = 0;
3704 #else
3705         do_mlockall = 1;
3706 #endif
3707
3708         argc -= diag;
3709         argv += diag;
3710         if (argc > 1)
3711                 launch_args_parse(argc, argv);
3712
3713         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3714                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3715                         strerror(errno));
3716         }
3717
3718         if (tx_first && interactive)
3719                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3720                                 "interactive mode.\n");
3721
3722         if (tx_first && lsc_interrupt) {
3723                 printf("Warning: lsc_interrupt needs to be off when "
3724                                 " using tx_first. Disabling.\n");
3725                 lsc_interrupt = 0;
3726         }
3727
3728         if (!nb_rxq && !nb_txq)
3729                 printf("Warning: Either rx or tx queues should be non-zero\n");
3730
3731         if (nb_rxq > 1 && nb_rxq > nb_txq)
3732                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3733                        "but nb_txq=%d will prevent to fully test it.\n",
3734                        nb_rxq, nb_txq);
3735
3736         init_config();
3737
3738         if (hot_plug) {
3739                 ret = rte_dev_hotplug_handle_enable();
3740                 if (ret) {
3741                         RTE_LOG(ERR, EAL,
3742                                 "fail to enable hotplug handling.");
3743                         return -1;
3744                 }
3745
3746                 ret = rte_dev_event_monitor_start();
3747                 if (ret) {
3748                         RTE_LOG(ERR, EAL,
3749                                 "fail to start device event monitoring.");
3750                         return -1;
3751                 }
3752
3753                 ret = rte_dev_event_callback_register(NULL,
3754                         dev_event_callback, NULL);
3755                 if (ret) {
3756                         RTE_LOG(ERR, EAL,
3757                                 "fail  to register device event callback\n");
3758                         return -1;
3759                 }
3760         }
3761
3762         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3763                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3764
3765         /* set all ports to promiscuous mode by default */
3766         RTE_ETH_FOREACH_DEV(port_id) {
3767                 ret = rte_eth_promiscuous_enable(port_id);
3768                 if (ret != 0)
3769                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3770                                 port_id, rte_strerror(-ret));
3771         }
3772
3773         /* Init metrics library */
3774         rte_metrics_init(rte_socket_id());
3775
3776 #ifdef RTE_LIBRTE_LATENCY_STATS
3777         if (latencystats_enabled != 0) {
3778                 int ret = rte_latencystats_init(1, NULL);
3779                 if (ret)
3780                         printf("Warning: latencystats init()"
3781                                 " returned error %d\n", ret);
3782                 printf("Latencystats running on lcore %d\n",
3783                         latencystats_lcore_id);
3784         }
3785 #endif
3786
3787         /* Setup bitrate stats */
3788 #ifdef RTE_LIBRTE_BITRATE
3789         if (bitrate_enabled != 0) {
3790                 bitrate_data = rte_stats_bitrate_create();
3791                 if (bitrate_data == NULL)
3792                         rte_exit(EXIT_FAILURE,
3793                                 "Could not allocate bitrate data.\n");
3794                 rte_stats_bitrate_reg(bitrate_data);
3795         }
3796 #endif
3797
3798 #ifdef RTE_LIBRTE_CMDLINE
3799         if (strlen(cmdline_filename) != 0)
3800                 cmdline_read_from_file(cmdline_filename);
3801
3802         if (interactive == 1) {
3803                 if (auto_start) {
3804                         printf("Start automatic packet forwarding\n");
3805                         start_packet_forwarding(0);
3806                 }
3807                 prompt();
3808                 pmd_test_exit();
3809         } else
3810 #endif
3811         {
3812                 char c;
3813                 int rc;
3814
3815                 f_quit = 0;
3816
3817                 printf("No commandline core given, start packet forwarding\n");
3818                 start_packet_forwarding(tx_first);
3819                 if (stats_period != 0) {
3820                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3821                         uint64_t timer_period;
3822
3823                         /* Convert to number of cycles */
3824                         timer_period = stats_period * rte_get_timer_hz();
3825
3826                         while (f_quit == 0) {
3827                                 cur_time = rte_get_timer_cycles();
3828                                 diff_time += cur_time - prev_time;
3829
3830                                 if (diff_time >= timer_period) {
3831                                         print_stats();
3832                                         /* Reset the timer */
3833                                         diff_time = 0;
3834                                 }
3835                                 /* Sleep to avoid unnecessary checks */
3836                                 prev_time = cur_time;
3837                                 sleep(1);
3838                         }
3839                 }
3840
3841                 printf("Press enter to exit\n");
3842                 rc = read(0, &c, 1);
3843                 pmd_test_exit();
3844                 if (rc < 0)
3845                         return 1;
3846         }
3847
3848         ret = rte_eal_cleanup();
3849         if (ret != 0)
3850                 rte_exit(EXIT_FAILURE,
3851                          "EAL cleanup failed: %s\n", strerror(-ret));
3852
3853         return EXIT_SUCCESS;
3854 }