app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_LIBRTE_IXGBE_PMD
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIBRTE_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIBRTE_BITRATE
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIBRTE_LATENCY_STATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use master core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182 #ifdef RTE_LIBRTE_IEEE1588
 183         &ieee1588_fwd_engine,
 184 #endif
 185         NULL,
 186 };
 187
 188 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES];
 189 uint16_t mempool_flags;
 190
 191 struct fwd_config cur_fwd_config;
 192 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 193 uint32_t retry_enabled;
 194 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 195 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 196
 197 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 198 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 199                                       * specified on command-line. */
 200 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 201
 202 /*
 203  * In container, it cannot terminate the process which running with 'stats-period'
 204  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 205  */
 206 uint8_t f_quit;
 207
 208 /*
 209  * Configuration of packet segments used by the "txonly" processing engine.
 210  */
 211 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 212 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 213         TXONLY_DEF_PACKET_LEN,
 214 };
 215 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 216
 217 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 218 /**< Split policy for packets to TX. */
 219
 220 uint8_t txonly_multi_flow;
 221 /**< Whether multiple flows are generated in TXONLY mode. */
 222
 223 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 224 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 225
 226 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 227 uint8_t dcb_config = 0;
 228
 229 /* Whether the dcb is in testing status */
 230 uint8_t dcb_test = 0;
 231
 232 /*
 233  * Configurable number of RX/TX queues.
 234  */
 235 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 236 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 237 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 238
 239 /*
 240  * Configurable number of RX/TX ring descriptors.
 241  * Defaults are supplied by drivers via ethdev.
 242  */
 243 #define RTE_TEST_RX_DESC_DEFAULT 0
 244 #define RTE_TEST_TX_DESC_DEFAULT 0
 245 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 246 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 247
 248 #define RTE_PMD_PARAM_UNSET -1
 249 /*
 250  * Configurable values of RX and TX ring threshold registers.
 251  */
 252
 253 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 254 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 255 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 256
 257 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 258 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 259 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 260
 261 /*
 262  * Configurable value of RX free threshold.
 263  */
 264 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 265
 266 /*
 267  * Configurable value of RX drop enable.
 268  */
 269 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 270
 271 /*
 272  * Configurable value of TX free threshold.
 273  */
 274 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 275
 276 /*
 277  * Configurable value of TX RS bit threshold.
 278  */
 279 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 280
 281 /*
 282  * Configurable value of buffered packets before sending.
 283  */
 284 uint16_t noisy_tx_sw_bufsz;
 285
 286 /*
 287  * Configurable value of packet buffer timeout.
 288  */
 289 uint16_t noisy_tx_sw_buf_flush_time;
 290
 291 /*
 292  * Configurable value for size of VNF internal memory area
 293  * used for simulating noisy neighbour behaviour
 294  */
 295 uint64_t noisy_lkup_mem_sz;
 296
 297 /*
 298  * Configurable value of number of random writes done in
 299  * VNF simulation memory area.
 300  */
 301 uint64_t noisy_lkup_num_writes;
 302
 303 /*
 304  * Configurable value of number of random reads done in
 305  * VNF simulation memory area.
 306  */
 307 uint64_t noisy_lkup_num_reads;
 308
 309 /*
 310  * Configurable value of number of random reads/writes done in
 311  * VNF simulation memory area.
 312  */
 313 uint64_t noisy_lkup_num_reads_writes;
 314
 315 /*
 316  * Receive Side Scaling (RSS) configuration.
 317  */
 318 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 319
 320 /*
 321  * Port topology configuration
 322  */
 323 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 324
 325 /*
 326  * Avoids to flush all the RX streams before starts forwarding.
 327  */
 328 uint8_t no_flush_rx = 0; /* flush by default */
 329
 330 /*
 331  * Flow API isolated mode.
 332  */
 333 uint8_t flow_isolate_all;
 334
 335 /*
 336  * Avoids to check link status when starting/stopping a port.
 337  */
 338 uint8_t no_link_check = 0; /* check by default */
 339
 340 /*
 341  * Don't automatically start all ports in interactive mode.
 342  */
 343 uint8_t no_device_start = 0;
 344
 345 /*
 346  * Enable link status change notification
 347  */
 348 uint8_t lsc_interrupt = 1; /* enabled by default */
 349
 350 /*
 351  * Enable device removal notification.
 352  */
 353 uint8_t rmv_interrupt = 1; /* enabled by default */
 354
 355 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 356
 357 /* After attach, port setup is called on event or by iterator */
 358 bool setup_on_probe_event = true;
 359
 360 /* Clear ptypes on port initialization. */
 361 uint8_t clear_ptypes = true;
 362
 363 /* Pretty printing of ethdev events */
 364 static const char * const eth_event_desc[] = {
 365         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 366         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 367         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 368         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 369         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 370         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 371         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 372         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 373         [RTE_ETH_EVENT_NEW] = "device probed",
 374         [RTE_ETH_EVENT_DESTROY] = "device released",
 375         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 376         [RTE_ETH_EVENT_MAX] = NULL,
 377 };
 378
 379 /*
 380  * Display or mask ether events
 381  * Default to all events except VF_MBOX
 382  */
 383 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 384                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 385                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 386                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 387                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 388                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 389                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 390                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 391 /*
 392  * Decide if all memory are locked for performance.
 393  */
 394 int do_mlockall = 0;
 395
 396 /*
 397  * NIC bypass mode configuration options.
 398  */
 399
 400 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
 401 /* The NIC bypass watchdog timeout. */
 402 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 403 #endif
 404
 405
 406 #ifdef RTE_LIBRTE_LATENCY_STATS
 407
 408 /*
 409  * Set when latency stats is enabled in the commandline
 410  */
 411 uint8_t latencystats_enabled;
 412
 413 /*
 414  * Lcore ID to serive latency statistics.
 415  */
 416 lcoreid_t latencystats_lcore_id = -1;
 417
 418 #endif
 419
 420 /*
 421  * Ethernet device configuration.
 422  */
 423 struct rte_eth_rxmode rx_mode = {
 424         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 425                 /**< Default maximum frame length. */
 426 };
 427
 428 struct rte_eth_txmode tx_mode = {
 429         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 430 };
 431
 432 struct rte_fdir_conf fdir_conf = {
 433         .mode = RTE_FDIR_MODE_NONE,
 434         .pballoc = RTE_FDIR_PBALLOC_64K,
 435         .status = RTE_FDIR_REPORT_STATUS,
 436         .mask = {
 437                 .vlan_tci_mask = 0xFFEF,
 438                 .ipv4_mask     = {
 439                         .src_ip = 0xFFFFFFFF,
 440                         .dst_ip = 0xFFFFFFFF,
 441                 },
 442                 .ipv6_mask     = {
 443                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 444                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 445                 },
 446                 .src_port_mask = 0xFFFF,
 447                 .dst_port_mask = 0xFFFF,
 448                 .mac_addr_byte_mask = 0xFF,
 449                 .tunnel_type_mask = 1,
 450                 .tunnel_id_mask = 0xFFFFFFFF,
 451         },
 452         .drop_queue = 127,
 453 };
 454
 455 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 456
 457 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
 458 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
 459
 460 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
 461 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
 462
 463 uint16_t nb_tx_queue_stats_mappings = 0;
 464 uint16_t nb_rx_queue_stats_mappings = 0;
 465
 466 /*
 467  * Display zero values by default for xstats
 468  */
 469 uint8_t xstats_hide_zero;
 470
 471 unsigned int num_sockets = 0;
 472 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 473
 474 #ifdef RTE_LIBRTE_BITRATE
 475 /* Bitrate statistics */
 476 struct rte_stats_bitrates *bitrate_data;
 477 lcoreid_t bitrate_lcore_id;
 478 uint8_t bitrate_enabled;
 479 #endif
 480
 481 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 482 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 483
 484 /*
 485  * hexadecimal bitmask of RX mq mode can be enabled.
 486  */
 487 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 488
 489 /* Forward function declarations */
 490 static void setup_attached_port(portid_t pi);
 491 static void map_port_queue_stats_mapping_registers(portid_t pi,
 492                                                    struct rte_port *port);
 493 static void check_all_ports_link_status(uint32_t port_mask);
 494 static int eth_event_callback(portid_t port_id,
 495                               enum rte_eth_event_type type,
 496                               void *param, void *ret_param);
 497 static void dev_event_callback(const char *device_name,
 498                                 enum rte_dev_event_type type,
 499                                 void *param);
 500
 501 /*
 502  * Check if all the ports are started.
 503  * If yes, return positive value. If not, return zero.
 504  */
 505 static int all_ports_started(void);
 506
 507 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 508 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 509
 510 /* Holds the registered mbuf dynamic flags names. */
 511 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 512
 513 /*
 514  * Helper function to check if socket is already discovered.
 515  * If yes, return positive value. If not, return zero.
 516  */
 517 int
 518 new_socket_id(unsigned int socket_id)
 519 {
 520         unsigned int i;
 521
 522         for (i = 0; i < num_sockets; i++) {
 523                 if (socket_ids[i] == socket_id)
 524                         return 0;
 525         }
 526         return 1;
 527 }
 528
 529 /*
 530  * Setup default configuration.
 531  */
 532 static void
 533 set_default_fwd_lcores_config(void)
 534 {
 535         unsigned int i;
 536         unsigned int nb_lc;
 537         unsigned int sock_num;
 538
 539         nb_lc = 0;
 540         for (i = 0; i < RTE_MAX_LCORE; i++) {
 541                 if (!rte_lcore_is_enabled(i))
 542                         continue;
 543                 sock_num = rte_lcore_to_socket_id(i);
 544                 if (new_socket_id(sock_num)) {
 545                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 546                                 rte_exit(EXIT_FAILURE,
 547                                          "Total sockets greater than %u\n",
 548                                          RTE_MAX_NUMA_NODES);
 549                         }
 550                         socket_ids[num_sockets++] = sock_num;
 551                 }
 552                 if (i == rte_get_master_lcore())
 553                         continue;
 554                 fwd_lcores_cpuids[nb_lc++] = i;
 555         }
 556         nb_lcores = (lcoreid_t) nb_lc;
 557         nb_cfg_lcores = nb_lcores;
 558         nb_fwd_lcores = 1;
 559 }
 560
 561 static void
 562 set_def_peer_eth_addrs(void)
 563 {
 564         portid_t i;
 565
 566         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 567                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 568                 peer_eth_addrs[i].addr_bytes[5] = i;
 569         }
 570 }
 571
 572 static void
 573 set_default_fwd_ports_config(void)
 574 {
 575         portid_t pt_id;
 576         int i = 0;
 577
 578         RTE_ETH_FOREACH_DEV(pt_id) {
 579                 fwd_ports_ids[i++] = pt_id;
 580
 581                 /* Update sockets info according to the attached device */
 582                 int socket_id = rte_eth_dev_socket_id(pt_id);
 583                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 584                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 585                                 rte_exit(EXIT_FAILURE,
 586                                          "Total sockets greater than %u\n",
 587                                          RTE_MAX_NUMA_NODES);
 588                         }
 589                         socket_ids[num_sockets++] = socket_id;
 590                 }
 591         }
 592
 593         nb_cfg_ports = nb_ports;
 594         nb_fwd_ports = nb_ports;
 595 }
 596
 597 void
 598 set_def_fwd_config(void)
 599 {
 600         set_default_fwd_lcores_config();
 601         set_def_peer_eth_addrs();
 602         set_default_fwd_ports_config();
 603 }
 604
 605 /* extremely pessimistic estimation of memory required to create a mempool */
 606 static int
 607 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 608 {
 609         unsigned int n_pages, mbuf_per_pg, leftover;
 610         uint64_t total_mem, mbuf_mem, obj_sz;
 611
 612         /* there is no good way to predict how much space the mempool will
 613          * occupy because it will allocate chunks on the fly, and some of those
 614          * will come from default DPDK memory while some will come from our
 615          * external memory, so just assume 128MB will be enough for everyone.
 616          */
 617         uint64_t hdr_mem = 128 << 20;
 618
 619         /* account for possible non-contiguousness */
 620         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 621         if (obj_sz > pgsz) {
 622                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 623                 return -1;
 624         }
 625
 626         mbuf_per_pg = pgsz / obj_sz;
 627         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 628         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 629
 630         mbuf_mem = n_pages * pgsz;
 631
 632         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 633
 634         if (total_mem > SIZE_MAX) {
 635                 TESTPMD_LOG(ERR, "Memory size too big\n");
 636                 return -1;
 637         }
 638         *out = (size_t)total_mem;
 639
 640         return 0;
 641 }
 642
 643 static int
 644 pagesz_flags(uint64_t page_sz)
 645 {
 646         /* as per mmap() manpage, all page sizes are log2 of page size
 647          * shifted by MAP_HUGE_SHIFT
 648          */
 649         int log2 = rte_log2_u64(page_sz);
 650
 651         return (log2 << HUGE_SHIFT);
 652 }
 653
 654 static void *
 655 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 656 {
 657         void *addr;
 658         int flags;
 659
 660         /* allocate anonymous hugepages */
 661         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 662         if (huge)
 663                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 664
 665         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 666         if (addr == MAP_FAILED)
 667                 return NULL;
 668
 669         return addr;
 670 }
 671
 672 struct extmem_param {
 673         void *addr;
 674         size_t len;
 675         size_t pgsz;
 676         rte_iova_t *iova_table;
 677         unsigned int iova_table_len;
 678 };
 679
 680 static int
 681 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 682                 bool huge)
 683 {
 684         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 685                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 686         unsigned int cur_page, n_pages, pgsz_idx;
 687         size_t mem_sz, cur_pgsz;
 688         rte_iova_t *iovas = NULL;
 689         void *addr;
 690         int ret;
 691
 692         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 693                 /* skip anything that is too big */
 694                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 695                         continue;
 696
 697                 cur_pgsz = pgsizes[pgsz_idx];
 698
 699                 /* if we were told not to allocate hugepages, override */
 700                 if (!huge)
 701                         cur_pgsz = sysconf(_SC_PAGESIZE);
 702
 703                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 704                 if (ret < 0) {
 705                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 706                         return -1;
 707                 }
 708
 709                 /* allocate our memory */
 710                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 711
 712                 /* if we couldn't allocate memory with a specified page size,
 713                  * that doesn't mean we can't do it with other page sizes, so
 714                  * try another one.
 715                  */
 716                 if (addr == NULL)
 717                         continue;
 718
 719                 /* store IOVA addresses for every page in this memory area */
 720                 n_pages = mem_sz / cur_pgsz;
 721
 722                 iovas = malloc(sizeof(*iovas) * n_pages);
 723
 724                 if (iovas == NULL) {
 725                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 726                         goto fail;
 727                 }
 728                 /* lock memory if it's not huge pages */
 729                 if (!huge)
 730                         mlock(addr, mem_sz);
 731
 732                 /* populate IOVA addresses */
 733                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 734                         rte_iova_t iova;
 735                         size_t offset;
 736                         void *cur;
 737
 738                         offset = cur_pgsz * cur_page;
 739                         cur = RTE_PTR_ADD(addr, offset);
 740
 741                         /* touch the page before getting its IOVA */
 742                         *(volatile char *)cur = 0;
 743
 744                         iova = rte_mem_virt2iova(cur);
 745
 746                         iovas[cur_page] = iova;
 747                 }
 748
 749                 break;
 750         }
 751         /* if we couldn't allocate anything */
 752         if (iovas == NULL)
 753                 return -1;
 754
 755         param->addr = addr;
 756         param->len = mem_sz;
 757         param->pgsz = cur_pgsz;
 758         param->iova_table = iovas;
 759         param->iova_table_len = n_pages;
 760
 761         return 0;
 762 fail:
 763         if (iovas)
 764                 free(iovas);
 765         if (addr)
 766                 munmap(addr, mem_sz);
 767
 768         return -1;
 769 }
 770
 771 static int
 772 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 773 {
 774         struct extmem_param param;
 775         int socket_id, ret;
 776
 777         memset(&param, 0, sizeof(param));
 778
 779         /* check if our heap exists */
 780         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 781         if (socket_id < 0) {
 782                 /* create our heap */
 783                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 784                 if (ret < 0) {
 785                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 786                         return -1;
 787                 }
 788         }
 789
 790         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 791         if (ret < 0) {
 792                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 793                 return -1;
 794         }
 795
 796         /* we now have a valid memory area, so add it to heap */
 797         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 798                         param.addr, param.len, param.iova_table,
 799                         param.iova_table_len, param.pgsz);
 800
 801         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 802
 803         /* not needed any more */
 804         free(param.iova_table);
 805
 806         if (ret < 0) {
 807                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 808                 munmap(param.addr, param.len);
 809                 return -1;
 810         }
 811
 812         /* success */
 813
 814         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 815                         param.len >> 20);
 816
 817         return 0;
 818 }
 819 static void
 820 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 821              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 822 {
 823         uint16_t pid = 0;
 824         int ret;
 825
 826         RTE_ETH_FOREACH_DEV(pid) {
 827                 struct rte_eth_dev *dev =
 828                         &rte_eth_devices[pid];
 829
 830                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 831                                         memhdr->len);
 832                 if (ret) {
 833                         TESTPMD_LOG(DEBUG,
 834                                     "unable to DMA unmap addr 0x%p "
 835                                     "for device %s\n",
 836                                     memhdr->addr, dev->data->name);
 837                 }
 838         }
 839         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 840         if (ret) {
 841                 TESTPMD_LOG(DEBUG,
 842                             "unable to un-register addr 0x%p\n", memhdr->addr);
 843         }
 844 }
 845
 846 static void
 847 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 848            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 849 {
 850         uint16_t pid = 0;
 851         size_t page_size = sysconf(_SC_PAGESIZE);
 852         int ret;
 853
 854         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 855                                   page_size);
 856         if (ret) {
 857                 TESTPMD_LOG(DEBUG,
 858                             "unable to register addr 0x%p\n", memhdr->addr);
 859                 return;
 860         }
 861         RTE_ETH_FOREACH_DEV(pid) {
 862                 struct rte_eth_dev *dev =
 863                         &rte_eth_devices[pid];
 864
 865                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 866                                       memhdr->len);
 867                 if (ret) {
 868                         TESTPMD_LOG(DEBUG,
 869                                     "unable to DMA map addr 0x%p "
 870                                     "for device %s\n",
 871                                     memhdr->addr, dev->data->name);
 872                 }
 873         }
 874 }
 875
 876 static unsigned int
 877 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 878             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 879 {
 880         struct rte_pktmbuf_extmem *xmem;
 881         unsigned int ext_num, zone_num, elt_num;
 882         uint16_t elt_size;
 883
 884         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 885         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 886         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 887
 888         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 889         if (xmem == NULL) {
 890                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 891                                  "external buffer descriptors\n");
 892                 *ext_mem = NULL;
 893                 return 0;
 894         }
 895         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 896                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 897                 const struct rte_memzone *mz;
 898                 char mz_name[RTE_MEMZONE_NAMESIZE];
 899                 int ret;
 900
 901                 ret = snprintf(mz_name, sizeof(mz_name),
 902                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 903                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 904                         errno = ENAMETOOLONG;
 905                         ext_num = 0;
 906                         break;
 907                 }
 908                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 909                                                  socket_id,
 910                                                  RTE_MEMZONE_IOVA_CONTIG |
 911                                                  RTE_MEMZONE_1GB |
 912                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 913                                                  EXTBUF_ZONE_SIZE);
 914                 if (mz == NULL) {
 915                         /*
 916                          * The caller exits on external buffer creation
 917                          * error, so there is no need to free memzones.
 918                          */
 919                         errno = ENOMEM;
 920                         ext_num = 0;
 921                         break;
 922                 }
 923                 xseg->buf_ptr = mz->addr;
 924                 xseg->buf_iova = mz->iova;
 925                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 926                 xseg->elt_size = elt_size;
 927         }
 928         if (ext_num == 0 && xmem != NULL) {
 929                 free(xmem);
 930                 xmem = NULL;
 931         }
 932         *ext_mem = xmem;
 933         return ext_num;
 934 }
 935
 936 /*
 937  * Configuration initialisation done once at init time.
 938  */
 939 static struct rte_mempool *
 940 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 941                  unsigned int socket_id)
 942 {
 943         char pool_name[RTE_MEMPOOL_NAMESIZE];
 944         struct rte_mempool *rte_mp = NULL;
 945         uint32_t mb_size;
 946
 947         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 948         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 949
 950         TESTPMD_LOG(INFO,
 951                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 952                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 953
 954         switch (mp_alloc_type) {
 955         case MP_ALLOC_NATIVE:
 956                 {
 957                         /* wrapper to rte_mempool_create() */
 958                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 959                                         rte_mbuf_best_mempool_ops());
 960                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 961                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 962                         break;
 963                 }
 964         case MP_ALLOC_ANON:
 965                 {
 966                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 967                                 mb_size, (unsigned int) mb_mempool_cache,
 968                                 sizeof(struct rte_pktmbuf_pool_private),
 969                                 socket_id, mempool_flags);
 970                         if (rte_mp == NULL)
 971                                 goto err;
 972
 973                         if (rte_mempool_populate_anon(rte_mp) == 0) {
 974                                 rte_mempool_free(rte_mp);
 975                                 rte_mp = NULL;
 976                                 goto err;
 977                         }
 978                         rte_pktmbuf_pool_init(rte_mp, NULL);
 979                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
 980                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
 981                         break;
 982                 }
 983         case MP_ALLOC_XMEM:
 984         case MP_ALLOC_XMEM_HUGE:
 985                 {
 986                         int heap_socket;
 987                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
 988
 989                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
 990                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
 991
 992                         heap_socket =
 993                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 994                         if (heap_socket < 0)
 995                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
 996
 997                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 998                                         rte_mbuf_best_mempool_ops());
 999                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1000                                         mb_mempool_cache, 0, mbuf_seg_size,
1001                                         heap_socket);
1002                         break;
1003                 }
1004         case MP_ALLOC_XBUF:
1005                 {
1006                         struct rte_pktmbuf_extmem *ext_mem;
1007                         unsigned int ext_num;
1008
1009                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1010                                                socket_id, pool_name, &ext_mem);
1011                         if (ext_num == 0)
1012                                 rte_exit(EXIT_FAILURE,
1013                                          "Can't create pinned data buffers\n");
1014
1015                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1016                                         rte_mbuf_best_mempool_ops());
1017                         rte_mp = rte_pktmbuf_pool_create_extbuf
1018                                         (pool_name, nb_mbuf, mb_mempool_cache,
1019                                          0, mbuf_seg_size, socket_id,
1020                                          ext_mem, ext_num);
1021                         free(ext_mem);
1022                         break;
1023                 }
1024         default:
1025                 {
1026                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1027                 }
1028         }
1029
1030 err:
1031         if (rte_mp == NULL) {
1032                 rte_exit(EXIT_FAILURE,
1033                         "Creation of mbuf pool for socket %u failed: %s\n",
1034                         socket_id, rte_strerror(rte_errno));
1035         } else if (verbose_level > 0) {
1036                 rte_mempool_dump(stdout, rte_mp);
1037         }
1038         return rte_mp;
1039 }
1040
1041 /*
1042  * Check given socket id is valid or not with NUMA mode,
1043  * if valid, return 0, else return -1
1044  */
1045 static int
1046 check_socket_id(const unsigned int socket_id)
1047 {
1048         static int warning_once = 0;
1049
1050         if (new_socket_id(socket_id)) {
1051                 if (!warning_once && numa_support)
1052                         printf("Warning: NUMA should be configured manually by"
1053                                " using --port-numa-config and"
1054                                " --ring-numa-config parameters along with"
1055                                " --numa.\n");
1056                 warning_once = 1;
1057                 return -1;
1058         }
1059         return 0;
1060 }
1061
1062 /*
1063  * Get the allowed maximum number of RX queues.
1064  * *pid return the port id which has minimal value of
1065  * max_rx_queues in all ports.
1066  */
1067 queueid_t
1068 get_allowed_max_nb_rxq(portid_t *pid)
1069 {
1070         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1071         bool max_rxq_valid = false;
1072         portid_t pi;
1073         struct rte_eth_dev_info dev_info;
1074
1075         RTE_ETH_FOREACH_DEV(pi) {
1076                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1077                         continue;
1078
1079                 max_rxq_valid = true;
1080                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1081                         allowed_max_rxq = dev_info.max_rx_queues;
1082                         *pid = pi;
1083                 }
1084         }
1085         return max_rxq_valid ? allowed_max_rxq : 0;
1086 }
1087
1088 /*
1089  * Check input rxq is valid or not.
1090  * If input rxq is not greater than any of maximum number
1091  * of RX queues of all ports, it is valid.
1092  * if valid, return 0, else return -1
1093  */
1094 int
1095 check_nb_rxq(queueid_t rxq)
1096 {
1097         queueid_t allowed_max_rxq;
1098         portid_t pid = 0;
1099
1100         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1101         if (rxq > allowed_max_rxq) {
1102                 printf("Fail: input rxq (%u) can't be greater "
1103                        "than max_rx_queues (%u) of port %u\n",
1104                        rxq,
1105                        allowed_max_rxq,
1106                        pid);
1107                 return -1;
1108         }
1109         return 0;
1110 }
1111
1112 /*
1113  * Get the allowed maximum number of TX queues.
1114  * *pid return the port id which has minimal value of
1115  * max_tx_queues in all ports.
1116  */
1117 queueid_t
1118 get_allowed_max_nb_txq(portid_t *pid)
1119 {
1120         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1121         bool max_txq_valid = false;
1122         portid_t pi;
1123         struct rte_eth_dev_info dev_info;
1124
1125         RTE_ETH_FOREACH_DEV(pi) {
1126                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1127                         continue;
1128
1129                 max_txq_valid = true;
1130                 if (dev_info.max_tx_queues < allowed_max_txq) {
1131                         allowed_max_txq = dev_info.max_tx_queues;
1132                         *pid = pi;
1133                 }
1134         }
1135         return max_txq_valid ? allowed_max_txq : 0;
1136 }
1137
1138 /*
1139  * Check input txq is valid or not.
1140  * If input txq is not greater than any of maximum number
1141  * of TX queues of all ports, it is valid.
1142  * if valid, return 0, else return -1
1143  */
1144 int
1145 check_nb_txq(queueid_t txq)
1146 {
1147         queueid_t allowed_max_txq;
1148         portid_t pid = 0;
1149
1150         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1151         if (txq > allowed_max_txq) {
1152                 printf("Fail: input txq (%u) can't be greater "
1153                        "than max_tx_queues (%u) of port %u\n",
1154                        txq,
1155                        allowed_max_txq,
1156                        pid);
1157                 return -1;
1158         }
1159         return 0;
1160 }
1161
1162 /*
1163  * Get the allowed maximum number of RXDs of every rx queue.
1164  * *pid return the port id which has minimal value of
1165  * max_rxd in all queues of all ports.
1166  */
1167 static uint16_t
1168 get_allowed_max_nb_rxd(portid_t *pid)
1169 {
1170         uint16_t allowed_max_rxd = UINT16_MAX;
1171         portid_t pi;
1172         struct rte_eth_dev_info dev_info;
1173
1174         RTE_ETH_FOREACH_DEV(pi) {
1175                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1176                         continue;
1177
1178                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1179                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1180                         *pid = pi;
1181                 }
1182         }
1183         return allowed_max_rxd;
1184 }
1185
1186 /*
1187  * Get the allowed minimal number of RXDs of every rx queue.
1188  * *pid return the port id which has minimal value of
1189  * min_rxd in all queues of all ports.
1190  */
1191 static uint16_t
1192 get_allowed_min_nb_rxd(portid_t *pid)
1193 {
1194         uint16_t allowed_min_rxd = 0;
1195         portid_t pi;
1196         struct rte_eth_dev_info dev_info;
1197
1198         RTE_ETH_FOREACH_DEV(pi) {
1199                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1200                         continue;
1201
1202                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1203                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1204                         *pid = pi;
1205                 }
1206         }
1207
1208         return allowed_min_rxd;
1209 }
1210
1211 /*
1212  * Check input rxd is valid or not.
1213  * If input rxd is not greater than any of maximum number
1214  * of RXDs of every Rx queues and is not less than any of
1215  * minimal number of RXDs of every Rx queues, it is valid.
1216  * if valid, return 0, else return -1
1217  */
1218 int
1219 check_nb_rxd(queueid_t rxd)
1220 {
1221         uint16_t allowed_max_rxd;
1222         uint16_t allowed_min_rxd;
1223         portid_t pid = 0;
1224
1225         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1226         if (rxd > allowed_max_rxd) {
1227                 printf("Fail: input rxd (%u) can't be greater "
1228                        "than max_rxds (%u) of port %u\n",
1229                        rxd,
1230                        allowed_max_rxd,
1231                        pid);
1232                 return -1;
1233         }
1234
1235         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1236         if (rxd < allowed_min_rxd) {
1237                 printf("Fail: input rxd (%u) can't be less "
1238                        "than min_rxds (%u) of port %u\n",
1239                        rxd,
1240                        allowed_min_rxd,
1241                        pid);
1242                 return -1;
1243         }
1244
1245         return 0;
1246 }
1247
1248 /*
1249  * Get the allowed maximum number of TXDs of every rx queues.
1250  * *pid return the port id which has minimal value of
1251  * max_txd in every tx queue.
1252  */
1253 static uint16_t
1254 get_allowed_max_nb_txd(portid_t *pid)
1255 {
1256         uint16_t allowed_max_txd = UINT16_MAX;
1257         portid_t pi;
1258         struct rte_eth_dev_info dev_info;
1259
1260         RTE_ETH_FOREACH_DEV(pi) {
1261                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1262                         continue;
1263
1264                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1265                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1266                         *pid = pi;
1267                 }
1268         }
1269         return allowed_max_txd;
1270 }
1271
1272 /*
1273  * Get the allowed maximum number of TXDs of every tx queues.
1274  * *pid return the port id which has minimal value of
1275  * min_txd in every tx queue.
1276  */
1277 static uint16_t
1278 get_allowed_min_nb_txd(portid_t *pid)
1279 {
1280         uint16_t allowed_min_txd = 0;
1281         portid_t pi;
1282         struct rte_eth_dev_info dev_info;
1283
1284         RTE_ETH_FOREACH_DEV(pi) {
1285                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1286                         continue;
1287
1288                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1289                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1290                         *pid = pi;
1291                 }
1292         }
1293
1294         return allowed_min_txd;
1295 }
1296
1297 /*
1298  * Check input txd is valid or not.
1299  * If input txd is not greater than any of maximum number
1300  * of TXDs of every Rx queues, it is valid.
1301  * if valid, return 0, else return -1
1302  */
1303 int
1304 check_nb_txd(queueid_t txd)
1305 {
1306         uint16_t allowed_max_txd;
1307         uint16_t allowed_min_txd;
1308         portid_t pid = 0;
1309
1310         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1311         if (txd > allowed_max_txd) {
1312                 printf("Fail: input txd (%u) can't be greater "
1313                        "than max_txds (%u) of port %u\n",
1314                        txd,
1315                        allowed_max_txd,
1316                        pid);
1317                 return -1;
1318         }
1319
1320         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1321         if (txd < allowed_min_txd) {
1322                 printf("Fail: input txd (%u) can't be less "
1323                        "than min_txds (%u) of port %u\n",
1324                        txd,
1325                        allowed_min_txd,
1326                        pid);
1327                 return -1;
1328         }
1329         return 0;
1330 }
1331
1332
1333 /*
1334  * Get the allowed maximum number of hairpin queues.
1335  * *pid return the port id which has minimal value of
1336  * max_hairpin_queues in all ports.
1337  */
1338 queueid_t
1339 get_allowed_max_nb_hairpinq(portid_t *pid)
1340 {
1341         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1342         portid_t pi;
1343         struct rte_eth_hairpin_cap cap;
1344
1345         RTE_ETH_FOREACH_DEV(pi) {
1346                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1347                         *pid = pi;
1348                         return 0;
1349                 }
1350                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1351                         allowed_max_hairpinq = cap.max_nb_queues;
1352                         *pid = pi;
1353                 }
1354         }
1355         return allowed_max_hairpinq;
1356 }
1357
1358 /*
1359  * Check input hairpin is valid or not.
1360  * If input hairpin is not greater than any of maximum number
1361  * of hairpin queues of all ports, it is valid.
1362  * if valid, return 0, else return -1
1363  */
1364 int
1365 check_nb_hairpinq(queueid_t hairpinq)
1366 {
1367         queueid_t allowed_max_hairpinq;
1368         portid_t pid = 0;
1369
1370         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1371         if (hairpinq > allowed_max_hairpinq) {
1372                 printf("Fail: input hairpin (%u) can't be greater "
1373                        "than max_hairpin_queues (%u) of port %u\n",
1374                        hairpinq, allowed_max_hairpinq, pid);
1375                 return -1;
1376         }
1377         return 0;
1378 }
1379
1380 static void
1381 init_config(void)
1382 {
1383         portid_t pid;
1384         struct rte_port *port;
1385         struct rte_mempool *mbp;
1386         unsigned int nb_mbuf_per_pool;
1387         lcoreid_t  lc_id;
1388         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1389         struct rte_gro_param gro_param;
1390         uint32_t gso_types;
1391         uint16_t data_size;
1392         bool warning = 0;
1393         int k;
1394         int ret;
1395
1396         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1397
1398         /* Configuration of logical cores. */
1399         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1400                                 sizeof(struct fwd_lcore *) * nb_lcores,
1401                                 RTE_CACHE_LINE_SIZE);
1402         if (fwd_lcores == NULL) {
1403                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1404                                                         "failed\n", nb_lcores);
1405         }
1406         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1407                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1408                                                sizeof(struct fwd_lcore),
1409                                                RTE_CACHE_LINE_SIZE);
1410                 if (fwd_lcores[lc_id] == NULL) {
1411                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1412                                                                 "failed\n");
1413                 }
1414                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1415         }
1416
1417         RTE_ETH_FOREACH_DEV(pid) {
1418                 port = &ports[pid];
1419                 /* Apply default TxRx configuration for all ports */
1420                 port->dev_conf.txmode = tx_mode;
1421                 port->dev_conf.rxmode = rx_mode;
1422
1423                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1424                 if (ret != 0)
1425                         rte_exit(EXIT_FAILURE,
1426                                  "rte_eth_dev_info_get() failed\n");
1427
1428                 if (!(port->dev_info.tx_offload_capa &
1429                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1430                         port->dev_conf.txmode.offloads &=
1431                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1432                 if (numa_support) {
1433                         if (port_numa[pid] != NUMA_NO_CONFIG)
1434                                 port_per_socket[port_numa[pid]]++;
1435                         else {
1436                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1437
1438                                 /*
1439                                  * if socket_id is invalid,
1440                                  * set to the first available socket.
1441                                  */
1442                                 if (check_socket_id(socket_id) < 0)
1443                                         socket_id = socket_ids[0];
1444                                 port_per_socket[socket_id]++;
1445                         }
1446                 }
1447
1448                 /* Apply Rx offloads configuration */
1449                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1450                         port->rx_conf[k].offloads =
1451                                 port->dev_conf.rxmode.offloads;
1452                 /* Apply Tx offloads configuration */
1453                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1454                         port->tx_conf[k].offloads =
1455                                 port->dev_conf.txmode.offloads;
1456
1457                 /* set flag to initialize port/queue */
1458                 port->need_reconfig = 1;
1459                 port->need_reconfig_queues = 1;
1460                 port->tx_metadata = 0;
1461
1462                 /* Check for maximum number of segments per MTU. Accordingly
1463                  * update the mbuf data size.
1464                  */
1465                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1466                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1467                         data_size = rx_mode.max_rx_pkt_len /
1468                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1469
1470                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1471                                                         mbuf_data_size) {
1472                                 mbuf_data_size = data_size +
1473                                                  RTE_PKTMBUF_HEADROOM;
1474                                 warning = 1;
1475                         }
1476                 }
1477         }
1478
1479         if (warning)
1480                 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n",
1481                             mbuf_data_size);
1482
1483         /*
1484          * Create pools of mbuf.
1485          * If NUMA support is disabled, create a single pool of mbuf in
1486          * socket 0 memory by default.
1487          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1488          *
1489          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1490          * nb_txd can be configured at run time.
1491          */
1492         if (param_total_num_mbufs)
1493                 nb_mbuf_per_pool = param_total_num_mbufs;
1494         else {
1495                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1496                         (nb_lcores * mb_mempool_cache) +
1497                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1498                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1499         }
1500
1501         if (numa_support) {
1502                 uint8_t i;
1503
1504                 for (i = 0; i < num_sockets; i++)
1505                         mempools[i] = mbuf_pool_create(mbuf_data_size,
1506                                                        nb_mbuf_per_pool,
1507                                                        socket_ids[i]);
1508         } else {
1509                 if (socket_num == UMA_NO_CONFIG)
1510                         mempools[0] = mbuf_pool_create(mbuf_data_size,
1511                                                        nb_mbuf_per_pool, 0);
1512                 else
1513                         mempools[socket_num] = mbuf_pool_create
1514                                                         (mbuf_data_size,
1515                                                          nb_mbuf_per_pool,
1516                                                          socket_num);
1517         }
1518
1519         init_port_config();
1520
1521         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1522                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1523         /*
1524          * Records which Mbuf pool to use by each logical core, if needed.
1525          */
1526         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1527                 mbp = mbuf_pool_find(
1528                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]));
1529
1530                 if (mbp == NULL)
1531                         mbp = mbuf_pool_find(0);
1532                 fwd_lcores[lc_id]->mbp = mbp;
1533                 /* initialize GSO context */
1534                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1535                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1536                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1537                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1538                         RTE_ETHER_CRC_LEN;
1539                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1540         }
1541
1542         /* Configuration of packet forwarding streams. */
1543         if (init_fwd_streams() < 0)
1544                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1545
1546         fwd_config_setup();
1547
1548         /* create a gro context for each lcore */
1549         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1550         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1551         gro_param.max_item_per_flow = MAX_PKT_BURST;
1552         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1553                 gro_param.socket_id = rte_lcore_to_socket_id(
1554                                 fwd_lcores_cpuids[lc_id]);
1555                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1556                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1557                         rte_exit(EXIT_FAILURE,
1558                                         "rte_gro_ctx_create() failed\n");
1559                 }
1560         }
1561 }
1562
1563
1564 void
1565 reconfig(portid_t new_port_id, unsigned socket_id)
1566 {
1567         struct rte_port *port;
1568         int ret;
1569
1570         /* Reconfiguration of Ethernet ports. */
1571         port = &ports[new_port_id];
1572
1573         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1574         if (ret != 0)
1575                 return;
1576
1577         /* set flag to initialize port/queue */
1578         port->need_reconfig = 1;
1579         port->need_reconfig_queues = 1;
1580         port->socket_id = socket_id;
1581
1582         init_port_config();
1583 }
1584
1585
1586 int
1587 init_fwd_streams(void)
1588 {
1589         portid_t pid;
1590         struct rte_port *port;
1591         streamid_t sm_id, nb_fwd_streams_new;
1592         queueid_t q;
1593
1594         /* set socket id according to numa or not */
1595         RTE_ETH_FOREACH_DEV(pid) {
1596                 port = &ports[pid];
1597                 if (nb_rxq > port->dev_info.max_rx_queues) {
1598                         printf("Fail: nb_rxq(%d) is greater than "
1599                                 "max_rx_queues(%d)\n", nb_rxq,
1600                                 port->dev_info.max_rx_queues);
1601                         return -1;
1602                 }
1603                 if (nb_txq > port->dev_info.max_tx_queues) {
1604                         printf("Fail: nb_txq(%d) is greater than "
1605                                 "max_tx_queues(%d)\n", nb_txq,
1606                                 port->dev_info.max_tx_queues);
1607                         return -1;
1608                 }
1609                 if (numa_support) {
1610                         if (port_numa[pid] != NUMA_NO_CONFIG)
1611                                 port->socket_id = port_numa[pid];
1612                         else {
1613                                 port->socket_id = rte_eth_dev_socket_id(pid);
1614
1615                                 /*
1616                                  * if socket_id is invalid,
1617                                  * set to the first available socket.
1618                                  */
1619                                 if (check_socket_id(port->socket_id) < 0)
1620                                         port->socket_id = socket_ids[0];
1621                         }
1622                 }
1623                 else {
1624                         if (socket_num == UMA_NO_CONFIG)
1625                                 port->socket_id = 0;
1626                         else
1627                                 port->socket_id = socket_num;
1628                 }
1629         }
1630
1631         q = RTE_MAX(nb_rxq, nb_txq);
1632         if (q == 0) {
1633                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1634                 return -1;
1635         }
1636         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1637         if (nb_fwd_streams_new == nb_fwd_streams)
1638                 return 0;
1639         /* clear the old */
1640         if (fwd_streams != NULL) {
1641                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1642                         if (fwd_streams[sm_id] == NULL)
1643                                 continue;
1644                         rte_free(fwd_streams[sm_id]);
1645                         fwd_streams[sm_id] = NULL;
1646                 }
1647                 rte_free(fwd_streams);
1648                 fwd_streams = NULL;
1649         }
1650
1651         /* init new */
1652         nb_fwd_streams = nb_fwd_streams_new;
1653         if (nb_fwd_streams) {
1654                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1655                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1656                         RTE_CACHE_LINE_SIZE);
1657                 if (fwd_streams == NULL)
1658                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1659                                  " (struct fwd_stream *)) failed\n",
1660                                  nb_fwd_streams);
1661
1662                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1663                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1664                                 " struct fwd_stream", sizeof(struct fwd_stream),
1665                                 RTE_CACHE_LINE_SIZE);
1666                         if (fwd_streams[sm_id] == NULL)
1667                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1668                                          "(struct fwd_stream) failed\n");
1669                 }
1670         }
1671
1672         return 0;
1673 }
1674
1675 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1676 static void
1677 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1678 {
1679         uint64_t total_burst, sburst;
1680         uint64_t nb_burst;
1681         uint64_t burst_stats[4];
1682         uint16_t pktnb_stats[4];
1683         uint16_t nb_pkt;
1684         int burst_percent[4], sburstp;
1685         int i;
1686
1687         /*
1688          * First compute the total number of packet bursts and the
1689          * two highest numbers of bursts of the same number of packets.
1690          */
1691         memset(&burst_stats, 0x0, sizeof(burst_stats));
1692         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1693
1694         /* Show stats for 0 burst size always */
1695         total_burst = pbs->pkt_burst_spread[0];
1696         burst_stats[0] = pbs->pkt_burst_spread[0];
1697         pktnb_stats[0] = 0;
1698
1699         /* Find the next 2 burst sizes with highest occurrences. */
1700         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1701                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1702
1703                 if (nb_burst == 0)
1704                         continue;
1705
1706                 total_burst += nb_burst;
1707
1708                 if (nb_burst > burst_stats[1]) {
1709                         burst_stats[2] = burst_stats[1];
1710                         pktnb_stats[2] = pktnb_stats[1];
1711                         burst_stats[1] = nb_burst;
1712                         pktnb_stats[1] = nb_pkt;
1713                 } else if (nb_burst > burst_stats[2]) {
1714                         burst_stats[2] = nb_burst;
1715                         pktnb_stats[2] = nb_pkt;
1716                 }
1717         }
1718         if (total_burst == 0)
1719                 return;
1720
1721         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1722         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1723                 if (i == 3) {
1724                         printf("%d%% of other]\n", 100 - sburstp);
1725                         return;
1726                 }
1727
1728                 sburst += burst_stats[i];
1729                 if (sburst == total_burst) {
1730                         printf("%d%% of %d pkts]\n",
1731                                 100 - sburstp, (int) pktnb_stats[i]);
1732                         return;
1733                 }
1734
1735                 burst_percent[i] =
1736                         (double)burst_stats[i] / total_burst * 100;
1737                 printf("%d%% of %d pkts + ",
1738                         burst_percent[i], (int) pktnb_stats[i]);
1739                 sburstp += burst_percent[i];
1740         }
1741 }
1742 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */
1743
1744 static void
1745 fwd_stream_stats_display(streamid_t stream_id)
1746 {
1747         struct fwd_stream *fs;
1748         static const char *fwd_top_stats_border = "-------";
1749
1750         fs = fwd_streams[stream_id];
1751         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1752             (fs->fwd_dropped == 0))
1753                 return;
1754         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1755                "TX Port=%2d/Queue=%2d %s\n",
1756                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1757                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1758         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1759                " TX-dropped: %-14"PRIu64,
1760                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1761
1762         /* if checksum mode */
1763         if (cur_fwd_eng == &csum_fwd_engine) {
1764                 printf("  RX- bad IP checksum: %-14"PRIu64
1765                        "  Rx- bad L4 checksum: %-14"PRIu64
1766                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1767                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1768                         fs->rx_bad_outer_l4_csum);
1769         } else {
1770                 printf("\n");
1771         }
1772
1773 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1774         pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1775         pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1776 #endif
1777 }
1778
1779 void
1780 fwd_stats_display(void)
1781 {
1782         static const char *fwd_stats_border = "----------------------";
1783         static const char *acc_stats_border = "+++++++++++++++";
1784         struct {
1785                 struct fwd_stream *rx_stream;
1786                 struct fwd_stream *tx_stream;
1787                 uint64_t tx_dropped;
1788                 uint64_t rx_bad_ip_csum;
1789                 uint64_t rx_bad_l4_csum;
1790                 uint64_t rx_bad_outer_l4_csum;
1791         } ports_stats[RTE_MAX_ETHPORTS];
1792         uint64_t total_rx_dropped = 0;
1793         uint64_t total_tx_dropped = 0;
1794         uint64_t total_rx_nombuf = 0;
1795         struct rte_eth_stats stats;
1796 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1797         uint64_t fwd_cycles = 0;
1798 #endif
1799         uint64_t total_recv = 0;
1800         uint64_t total_xmit = 0;
1801         struct rte_port *port;
1802         streamid_t sm_id;
1803         portid_t pt_id;
1804         int i;
1805
1806         memset(ports_stats, 0, sizeof(ports_stats));
1807
1808         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1809                 struct fwd_stream *fs = fwd_streams[sm_id];
1810
1811                 if (cur_fwd_config.nb_fwd_streams >
1812                     cur_fwd_config.nb_fwd_ports) {
1813                         fwd_stream_stats_display(sm_id);
1814                 } else {
1815                         ports_stats[fs->tx_port].tx_stream = fs;
1816                         ports_stats[fs->rx_port].rx_stream = fs;
1817                 }
1818
1819                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1820
1821                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1822                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1823                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1824                                 fs->rx_bad_outer_l4_csum;
1825
1826 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1827                 fwd_cycles += fs->core_cycles;
1828 #endif
1829         }
1830         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1831                 uint8_t j;
1832
1833                 pt_id = fwd_ports_ids[i];
1834                 port = &ports[pt_id];
1835
1836                 rte_eth_stats_get(pt_id, &stats);
1837                 stats.ipackets -= port->stats.ipackets;
1838                 stats.opackets -= port->stats.opackets;
1839                 stats.ibytes -= port->stats.ibytes;
1840                 stats.obytes -= port->stats.obytes;
1841                 stats.imissed -= port->stats.imissed;
1842                 stats.oerrors -= port->stats.oerrors;
1843                 stats.rx_nombuf -= port->stats.rx_nombuf;
1844
1845                 total_recv += stats.ipackets;
1846                 total_xmit += stats.opackets;
1847                 total_rx_dropped += stats.imissed;
1848                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1849                 total_tx_dropped += stats.oerrors;
1850                 total_rx_nombuf  += stats.rx_nombuf;
1851
1852                 printf("\n  %s Forward statistics for port %-2d %s\n",
1853                        fwd_stats_border, pt_id, fwd_stats_border);
1854
1855                 if (!port->rx_queue_stats_mapping_enabled &&
1856                     !port->tx_queue_stats_mapping_enabled) {
1857                         printf("  RX-packets: %-14"PRIu64
1858                                " RX-dropped: %-14"PRIu64
1859                                "RX-total: %-"PRIu64"\n",
1860                                stats.ipackets, stats.imissed,
1861                                stats.ipackets + stats.imissed);
1862
1863                         if (cur_fwd_eng == &csum_fwd_engine)
1864                                 printf("  Bad-ipcsum: %-14"PRIu64
1865                                        " Bad-l4csum: %-14"PRIu64
1866                                        "Bad-outer-l4csum: %-14"PRIu64"\n",
1867                                        ports_stats[pt_id].rx_bad_ip_csum,
1868                                        ports_stats[pt_id].rx_bad_l4_csum,
1869                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1870                         if (stats.ierrors + stats.rx_nombuf > 0) {
1871                                 printf("  RX-error: %-"PRIu64"\n",
1872                                        stats.ierrors);
1873                                 printf("  RX-nombufs: %-14"PRIu64"\n",
1874                                        stats.rx_nombuf);
1875                         }
1876
1877                         printf("  TX-packets: %-14"PRIu64
1878                                " TX-dropped: %-14"PRIu64
1879                                "TX-total: %-"PRIu64"\n",
1880                                stats.opackets, ports_stats[pt_id].tx_dropped,
1881                                stats.opackets + ports_stats[pt_id].tx_dropped);
1882                 } else {
1883                         printf("  RX-packets:             %14"PRIu64
1884                                "    RX-dropped:%14"PRIu64
1885                                "    RX-total:%14"PRIu64"\n",
1886                                stats.ipackets, stats.imissed,
1887                                stats.ipackets + stats.imissed);
1888
1889                         if (cur_fwd_eng == &csum_fwd_engine)
1890                                 printf("  Bad-ipcsum:%14"PRIu64
1891                                        "    Bad-l4csum:%14"PRIu64
1892                                        "    Bad-outer-l4csum: %-14"PRIu64"\n",
1893                                        ports_stats[pt_id].rx_bad_ip_csum,
1894                                        ports_stats[pt_id].rx_bad_l4_csum,
1895                                        ports_stats[pt_id].rx_bad_outer_l4_csum);
1896                         if ((stats.ierrors + stats.rx_nombuf) > 0) {
1897                                 printf("  RX-error:%"PRIu64"\n", stats.ierrors);
1898                                 printf("  RX-nombufs:             %14"PRIu64"\n",
1899                                        stats.rx_nombuf);
1900                         }
1901
1902                         printf("  TX-packets:             %14"PRIu64
1903                                "    TX-dropped:%14"PRIu64
1904                                "    TX-total:%14"PRIu64"\n",
1905                                stats.opackets, ports_stats[pt_id].tx_dropped,
1906                                stats.opackets + ports_stats[pt_id].tx_dropped);
1907                 }
1908
1909 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1910                 if (ports_stats[pt_id].rx_stream)
1911                         pkt_burst_stats_display("RX",
1912                                 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1913                 if (ports_stats[pt_id].tx_stream)
1914                         pkt_burst_stats_display("TX",
1915                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1916 #endif
1917
1918                 if (port->rx_queue_stats_mapping_enabled) {
1919                         printf("\n");
1920                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1921                                 printf("  Stats reg %2d RX-packets:%14"PRIu64
1922                                        "     RX-errors:%14"PRIu64
1923                                        "    RX-bytes:%14"PRIu64"\n",
1924                                        j, stats.q_ipackets[j],
1925                                        stats.q_errors[j], stats.q_ibytes[j]);
1926                         }
1927                         printf("\n");
1928                 }
1929                 if (port->tx_queue_stats_mapping_enabled) {
1930                         for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1931                                 printf("  Stats reg %2d TX-packets:%14"PRIu64
1932                                        "                                 TX-bytes:%14"
1933                                        PRIu64"\n",
1934                                        j, stats.q_opackets[j],
1935                                        stats.q_obytes[j]);
1936                         }
1937                 }
1938
1939                 printf("  %s--------------------------------%s\n",
1940                        fwd_stats_border, fwd_stats_border);
1941         }
1942
1943         printf("\n  %s Accumulated forward statistics for all ports"
1944                "%s\n",
1945                acc_stats_border, acc_stats_border);
1946         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1947                "%-"PRIu64"\n"
1948                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1949                "%-"PRIu64"\n",
1950                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1951                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1952         if (total_rx_nombuf > 0)
1953                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1954         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1955                "%s\n",
1956                acc_stats_border, acc_stats_border);
1957 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1958 #define CYC_PER_MHZ 1E6
1959         if (total_recv > 0)
1960                 printf("\n  CPU cycles/packet=%.2F (total cycles="
1961                        "%"PRIu64" / total RX packets=%"PRIu64") at %"PRIu64
1962                        " MHz Clock\n",
1963                        (double) fwd_cycles / total_recv,
1964                        fwd_cycles, total_recv,
1965                        (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1966 #endif
1967 }
1968
1969 void
1970 fwd_stats_reset(void)
1971 {
1972         streamid_t sm_id;
1973         portid_t pt_id;
1974         int i;
1975
1976         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1977                 pt_id = fwd_ports_ids[i];
1978                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1979         }
1980         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1981                 struct fwd_stream *fs = fwd_streams[sm_id];
1982
1983                 fs->rx_packets = 0;
1984                 fs->tx_packets = 0;
1985                 fs->fwd_dropped = 0;
1986                 fs->rx_bad_ip_csum = 0;
1987                 fs->rx_bad_l4_csum = 0;
1988                 fs->rx_bad_outer_l4_csum = 0;
1989
1990 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
1991                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1992                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1993 #endif
1994 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
1995                 fs->core_cycles = 0;
1996 #endif
1997         }
1998 }
1999
2000 static void
2001 flush_fwd_rx_queues(void)
2002 {
2003         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2004         portid_t  rxp;
2005         portid_t port_id;
2006         queueid_t rxq;
2007         uint16_t  nb_rx;
2008         uint16_t  i;
2009         uint8_t   j;
2010         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2011         uint64_t timer_period;
2012
2013         /* convert to number of cycles */
2014         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2015
2016         for (j = 0; j < 2; j++) {
2017                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2018                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2019                                 port_id = fwd_ports_ids[rxp];
2020                                 /**
2021                                 * testpmd can stuck in the below do while loop
2022                                 * if rte_eth_rx_burst() always returns nonzero
2023                                 * packets. So timer is added to exit this loop
2024                                 * after 1sec timer expiry.
2025                                 */
2026                                 prev_tsc = rte_rdtsc();
2027                                 do {
2028                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2029                                                 pkts_burst, MAX_PKT_BURST);
2030                                         for (i = 0; i < nb_rx; i++)
2031                                                 rte_pktmbuf_free(pkts_burst[i]);
2032
2033                                         cur_tsc = rte_rdtsc();
2034                                         diff_tsc = cur_tsc - prev_tsc;
2035                                         timer_tsc += diff_tsc;
2036                                 } while ((nb_rx > 0) &&
2037                                         (timer_tsc < timer_period));
2038                                 timer_tsc = 0;
2039                         }
2040                 }
2041                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2042         }
2043 }
2044
2045 static void
2046 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2047 {
2048         struct fwd_stream **fsm;
2049         streamid_t nb_fs;
2050         streamid_t sm_id;
2051 #ifdef RTE_LIBRTE_BITRATE
2052         uint64_t tics_per_1sec;
2053         uint64_t tics_datum;
2054         uint64_t tics_current;
2055         uint16_t i, cnt_ports;
2056
2057         cnt_ports = nb_ports;
2058         tics_datum = rte_rdtsc();
2059         tics_per_1sec = rte_get_timer_hz();
2060 #endif
2061         fsm = &fwd_streams[fc->stream_idx];
2062         nb_fs = fc->stream_nb;
2063         do {
2064                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2065                         (*pkt_fwd)(fsm[sm_id]);
2066 #ifdef RTE_LIBRTE_BITRATE
2067                 if (bitrate_enabled != 0 &&
2068                                 bitrate_lcore_id == rte_lcore_id()) {
2069                         tics_current = rte_rdtsc();
2070                         if (tics_current - tics_datum >= tics_per_1sec) {
2071                                 /* Periodic bitrate calculation */
2072                                 for (i = 0; i < cnt_ports; i++)
2073                                         rte_stats_bitrate_calc(bitrate_data,
2074                                                 ports_ids[i]);
2075                                 tics_datum = tics_current;
2076                         }
2077                 }
2078 #endif
2079 #ifdef RTE_LIBRTE_LATENCY_STATS
2080                 if (latencystats_enabled != 0 &&
2081                                 latencystats_lcore_id == rte_lcore_id())
2082                         rte_latencystats_update();
2083 #endif
2084
2085         } while (! fc->stopped);
2086 }
2087
2088 static int
2089 start_pkt_forward_on_core(void *fwd_arg)
2090 {
2091         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2092                              cur_fwd_config.fwd_eng->packet_fwd);
2093         return 0;
2094 }
2095
2096 /*
2097  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2098  * Used to start communication flows in network loopback test configurations.
2099  */
2100 static int
2101 run_one_txonly_burst_on_core(void *fwd_arg)
2102 {
2103         struct fwd_lcore *fwd_lc;
2104         struct fwd_lcore tmp_lcore;
2105
2106         fwd_lc = (struct fwd_lcore *) fwd_arg;
2107         tmp_lcore = *fwd_lc;
2108         tmp_lcore.stopped = 1;
2109         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2110         return 0;
2111 }
2112
2113 /*
2114  * Launch packet forwarding:
2115  *     - Setup per-port forwarding context.
2116  *     - launch logical cores with their forwarding configuration.
2117  */
2118 static void
2119 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2120 {
2121         port_fwd_begin_t port_fwd_begin;
2122         unsigned int i;
2123         unsigned int lc_id;
2124         int diag;
2125
2126         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2127         if (port_fwd_begin != NULL) {
2128                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2129                         (*port_fwd_begin)(fwd_ports_ids[i]);
2130         }
2131         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2132                 lc_id = fwd_lcores_cpuids[i];
2133                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2134                         fwd_lcores[i]->stopped = 0;
2135                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2136                                                      fwd_lcores[i], lc_id);
2137                         if (diag != 0)
2138                                 printf("launch lcore %u failed - diag=%d\n",
2139                                        lc_id, diag);
2140                 }
2141         }
2142 }
2143
2144 /*
2145  * Launch packet forwarding configuration.
2146  */
2147 void
2148 start_packet_forwarding(int with_tx_first)
2149 {
2150         port_fwd_begin_t port_fwd_begin;
2151         port_fwd_end_t  port_fwd_end;
2152         struct rte_port *port;
2153         unsigned int i;
2154         portid_t   pt_id;
2155
2156         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2157                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2158
2159         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2160                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2161
2162         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2163                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2164                 (!nb_rxq || !nb_txq))
2165                 rte_exit(EXIT_FAILURE,
2166                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2167                         cur_fwd_eng->fwd_mode_name);
2168
2169         if (all_ports_started() == 0) {
2170                 printf("Not all ports were started\n");
2171                 return;
2172         }
2173         if (test_done == 0) {
2174                 printf("Packet forwarding already started\n");
2175                 return;
2176         }
2177
2178
2179         if(dcb_test) {
2180                 for (i = 0; i < nb_fwd_ports; i++) {
2181                         pt_id = fwd_ports_ids[i];
2182                         port = &ports[pt_id];
2183                         if (!port->dcb_flag) {
2184                                 printf("In DCB mode, all forwarding ports must "
2185                                        "be configured in this mode.\n");
2186                                 return;
2187                         }
2188                 }
2189                 if (nb_fwd_lcores == 1) {
2190                         printf("In DCB mode,the nb forwarding cores "
2191                                "should be larger than 1.\n");
2192                         return;
2193                 }
2194         }
2195         test_done = 0;
2196
2197         fwd_config_setup();
2198
2199         if(!no_flush_rx)
2200                 flush_fwd_rx_queues();
2201
2202         pkt_fwd_config_display(&cur_fwd_config);
2203         rxtx_config_display();
2204
2205         fwd_stats_reset();
2206         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2207                 pt_id = fwd_ports_ids[i];
2208                 port = &ports[pt_id];
2209                 map_port_queue_stats_mapping_registers(pt_id, port);
2210         }
2211         if (with_tx_first) {
2212                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2213                 if (port_fwd_begin != NULL) {
2214                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2215                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2216                 }
2217                 while (with_tx_first--) {
2218                         launch_packet_forwarding(
2219                                         run_one_txonly_burst_on_core);
2220                         rte_eal_mp_wait_lcore();
2221                 }
2222                 port_fwd_end = tx_only_engine.port_fwd_end;
2223                 if (port_fwd_end != NULL) {
2224                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2225                                 (*port_fwd_end)(fwd_ports_ids[i]);
2226                 }
2227         }
2228         launch_packet_forwarding(start_pkt_forward_on_core);
2229 }
2230
2231 void
2232 stop_packet_forwarding(void)
2233 {
2234         port_fwd_end_t port_fwd_end;
2235         lcoreid_t lc_id;
2236         portid_t pt_id;
2237         int i;
2238
2239         if (test_done) {
2240                 printf("Packet forwarding not started\n");
2241                 return;
2242         }
2243         printf("Telling cores to stop...");
2244         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2245                 fwd_lcores[lc_id]->stopped = 1;
2246         printf("\nWaiting for lcores to finish...\n");
2247         rte_eal_mp_wait_lcore();
2248         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2249         if (port_fwd_end != NULL) {
2250                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2251                         pt_id = fwd_ports_ids[i];
2252                         (*port_fwd_end)(pt_id);
2253                 }
2254         }
2255
2256         fwd_stats_display();
2257
2258         printf("\nDone.\n");
2259         test_done = 1;
2260 }
2261
2262 void
2263 dev_set_link_up(portid_t pid)
2264 {
2265         if (rte_eth_dev_set_link_up(pid) < 0)
2266                 printf("\nSet link up fail.\n");
2267 }
2268
2269 void
2270 dev_set_link_down(portid_t pid)
2271 {
2272         if (rte_eth_dev_set_link_down(pid) < 0)
2273                 printf("\nSet link down fail.\n");
2274 }
2275
2276 static int
2277 all_ports_started(void)
2278 {
2279         portid_t pi;
2280         struct rte_port *port;
2281
2282         RTE_ETH_FOREACH_DEV(pi) {
2283                 port = &ports[pi];
2284                 /* Check if there is a port which is not started */
2285                 if ((port->port_status != RTE_PORT_STARTED) &&
2286                         (port->slave_flag == 0))
2287                         return 0;
2288         }
2289
2290         /* No port is not started */
2291         return 1;
2292 }
2293
2294 int
2295 port_is_stopped(portid_t port_id)
2296 {
2297         struct rte_port *port = &ports[port_id];
2298
2299         if ((port->port_status != RTE_PORT_STOPPED) &&
2300             (port->slave_flag == 0))
2301                 return 0;
2302         return 1;
2303 }
2304
2305 int
2306 all_ports_stopped(void)
2307 {
2308         portid_t pi;
2309
2310         RTE_ETH_FOREACH_DEV(pi) {
2311                 if (!port_is_stopped(pi))
2312                         return 0;
2313         }
2314
2315         return 1;
2316 }
2317
2318 int
2319 port_is_started(portid_t port_id)
2320 {
2321         if (port_id_is_invalid(port_id, ENABLED_WARN))
2322                 return 0;
2323
2324         if (ports[port_id].port_status != RTE_PORT_STARTED)
2325                 return 0;
2326
2327         return 1;
2328 }
2329
2330 /* Configure the Rx and Tx hairpin queues for the selected port. */
2331 static int
2332 setup_hairpin_queues(portid_t pi)
2333 {
2334         queueid_t qi;
2335         struct rte_eth_hairpin_conf hairpin_conf = {
2336                 .peer_count = 1,
2337         };
2338         int i;
2339         int diag;
2340         struct rte_port *port = &ports[pi];
2341
2342         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2343                 hairpin_conf.peers[0].port = pi;
2344                 hairpin_conf.peers[0].queue = i + nb_rxq;
2345                 diag = rte_eth_tx_hairpin_queue_setup
2346                         (pi, qi, nb_txd, &hairpin_conf);
2347                 i++;
2348                 if (diag == 0)
2349                         continue;
2350
2351                 /* Fail to setup rx queue, return */
2352                 if (rte_atomic16_cmpset(&(port->port_status),
2353                                         RTE_PORT_HANDLING,
2354                                         RTE_PORT_STOPPED) == 0)
2355                         printf("Port %d can not be set back "
2356                                         "to stopped\n", pi);
2357                 printf("Fail to configure port %d hairpin "
2358                                 "queues\n", pi);
2359                 /* try to reconfigure queues next time */
2360                 port->need_reconfig_queues = 1;
2361                 return -1;
2362         }
2363         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2364                 hairpin_conf.peers[0].port = pi;
2365                 hairpin_conf.peers[0].queue = i + nb_txq;
2366                 diag = rte_eth_rx_hairpin_queue_setup
2367                         (pi, qi, nb_rxd, &hairpin_conf);
2368                 i++;
2369                 if (diag == 0)
2370                         continue;
2371
2372                 /* Fail to setup rx queue, return */
2373                 if (rte_atomic16_cmpset(&(port->port_status),
2374                                         RTE_PORT_HANDLING,
2375                                         RTE_PORT_STOPPED) == 0)
2376                         printf("Port %d can not be set back "
2377                                         "to stopped\n", pi);
2378                 printf("Fail to configure port %d hairpin "
2379                                 "queues\n", pi);
2380                 /* try to reconfigure queues next time */
2381                 port->need_reconfig_queues = 1;
2382                 return -1;
2383         }
2384         return 0;
2385 }
2386
2387 int
2388 start_port(portid_t pid)
2389 {
2390         int diag, need_check_link_status = -1;
2391         portid_t pi;
2392         queueid_t qi;
2393         struct rte_port *port;
2394         struct rte_ether_addr mac_addr;
2395         struct rte_eth_hairpin_cap cap;
2396
2397         if (port_id_is_invalid(pid, ENABLED_WARN))
2398                 return 0;
2399
2400         if(dcb_config)
2401                 dcb_test = 1;
2402         RTE_ETH_FOREACH_DEV(pi) {
2403                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2404                         continue;
2405
2406                 need_check_link_status = 0;
2407                 port = &ports[pi];
2408                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2409                                                  RTE_PORT_HANDLING) == 0) {
2410                         printf("Port %d is now not stopped\n", pi);
2411                         continue;
2412                 }
2413
2414                 if (port->need_reconfig > 0) {
2415                         port->need_reconfig = 0;
2416
2417                         if (flow_isolate_all) {
2418                                 int ret = port_flow_isolate(pi, 1);
2419                                 if (ret) {
2420                                         printf("Failed to apply isolated"
2421                                                " mode on port %d\n", pi);
2422                                         return -1;
2423                                 }
2424                         }
2425                         configure_rxtx_dump_callbacks(0);
2426                         printf("Configuring Port %d (socket %u)\n", pi,
2427                                         port->socket_id);
2428                         if (nb_hairpinq > 0 &&
2429                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2430                                 printf("Port %d doesn't support hairpin "
2431                                        "queues\n", pi);
2432                                 return -1;
2433                         }
2434                         /* configure port */
2435                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2436                                                      nb_txq + nb_hairpinq,
2437                                                      &(port->dev_conf));
2438                         if (diag != 0) {
2439                                 if (rte_atomic16_cmpset(&(port->port_status),
2440                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2441                                         printf("Port %d can not be set back "
2442                                                         "to stopped\n", pi);
2443                                 printf("Fail to configure port %d\n", pi);
2444                                 /* try to reconfigure port next time */
2445                                 port->need_reconfig = 1;
2446                                 return -1;
2447                         }
2448                 }
2449                 if (port->need_reconfig_queues > 0) {
2450                         port->need_reconfig_queues = 0;
2451                         /* setup tx queues */
2452                         for (qi = 0; qi < nb_txq; qi++) {
2453                                 if ((numa_support) &&
2454                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2455                                         diag = rte_eth_tx_queue_setup(pi, qi,
2456                                                 port->nb_tx_desc[qi],
2457                                                 txring_numa[pi],
2458                                                 &(port->tx_conf[qi]));
2459                                 else
2460                                         diag = rte_eth_tx_queue_setup(pi, qi,
2461                                                 port->nb_tx_desc[qi],
2462                                                 port->socket_id,
2463                                                 &(port->tx_conf[qi]));
2464
2465                                 if (diag == 0)
2466                                         continue;
2467
2468                                 /* Fail to setup tx queue, return */
2469                                 if (rte_atomic16_cmpset(&(port->port_status),
2470                                                         RTE_PORT_HANDLING,
2471                                                         RTE_PORT_STOPPED) == 0)
2472                                         printf("Port %d can not be set back "
2473                                                         "to stopped\n", pi);
2474                                 printf("Fail to configure port %d tx queues\n",
2475                                        pi);
2476                                 /* try to reconfigure queues next time */
2477                                 port->need_reconfig_queues = 1;
2478                                 return -1;
2479                         }
2480                         for (qi = 0; qi < nb_rxq; qi++) {
2481                                 /* setup rx queues */
2482                                 if ((numa_support) &&
2483                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2484                                         struct rte_mempool * mp =
2485                                                 mbuf_pool_find(rxring_numa[pi]);
2486                                         if (mp == NULL) {
2487                                                 printf("Failed to setup RX queue:"
2488                                                         "No mempool allocation"
2489                                                         " on the socket %d\n",
2490                                                         rxring_numa[pi]);
2491                                                 return -1;
2492                                         }
2493
2494                                         diag = rte_eth_rx_queue_setup(pi, qi,
2495                                              port->nb_rx_desc[qi],
2496                                              rxring_numa[pi],
2497                                              &(port->rx_conf[qi]),
2498                                              mp);
2499                                 } else {
2500                                         struct rte_mempool *mp =
2501                                                 mbuf_pool_find(port->socket_id);
2502                                         if (mp == NULL) {
2503                                                 printf("Failed to setup RX queue:"
2504                                                         "No mempool allocation"
2505                                                         " on the socket %d\n",
2506                                                         port->socket_id);
2507                                                 return -1;
2508                                         }
2509                                         diag = rte_eth_rx_queue_setup(pi, qi,
2510                                              port->nb_rx_desc[qi],
2511                                              port->socket_id,
2512                                              &(port->rx_conf[qi]),
2513                                              mp);
2514                                 }
2515                                 if (diag == 0)
2516                                         continue;
2517
2518                                 /* Fail to setup rx queue, return */
2519                                 if (rte_atomic16_cmpset(&(port->port_status),
2520                                                         RTE_PORT_HANDLING,
2521                                                         RTE_PORT_STOPPED) == 0)
2522                                         printf("Port %d can not be set back "
2523                                                         "to stopped\n", pi);
2524                                 printf("Fail to configure port %d rx queues\n",
2525                                        pi);
2526                                 /* try to reconfigure queues next time */
2527                                 port->need_reconfig_queues = 1;
2528                                 return -1;
2529                         }
2530                         /* setup hairpin queues */
2531                         if (setup_hairpin_queues(pi) != 0)
2532                                 return -1;
2533                 }
2534                 configure_rxtx_dump_callbacks(verbose_level);
2535                 if (clear_ptypes) {
2536                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2537                                         NULL, 0);
2538                         if (diag < 0)
2539                                 printf(
2540                                 "Port %d: Failed to disable Ptype parsing\n",
2541                                 pi);
2542                 }
2543
2544                 /* start port */
2545                 if (rte_eth_dev_start(pi) < 0) {
2546                         printf("Fail to start port %d\n", pi);
2547
2548                         /* Fail to setup rx queue, return */
2549                         if (rte_atomic16_cmpset(&(port->port_status),
2550                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2551                                 printf("Port %d can not be set back to "
2552                                                         "stopped\n", pi);
2553                         continue;
2554                 }
2555
2556                 if (rte_atomic16_cmpset(&(port->port_status),
2557                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2558                         printf("Port %d can not be set into started\n", pi);
2559
2560                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2561                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2562                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2563                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2564                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2565
2566                 /* at least one port started, need checking link status */
2567                 need_check_link_status = 1;
2568         }
2569
2570         if (need_check_link_status == 1 && !no_link_check)
2571                 check_all_ports_link_status(RTE_PORT_ALL);
2572         else if (need_check_link_status == 0)
2573                 printf("Please stop the ports first\n");
2574
2575         printf("Done\n");
2576         return 0;
2577 }
2578
2579 void
2580 stop_port(portid_t pid)
2581 {
2582         portid_t pi;
2583         struct rte_port *port;
2584         int need_check_link_status = 0;
2585
2586         if (dcb_test) {
2587                 dcb_test = 0;
2588                 dcb_config = 0;
2589         }
2590
2591         if (port_id_is_invalid(pid, ENABLED_WARN))
2592                 return;
2593
2594         printf("Stopping ports...\n");
2595
2596         RTE_ETH_FOREACH_DEV(pi) {
2597                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2598                         continue;
2599
2600                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2601                         printf("Please remove port %d from forwarding configuration.\n", pi);
2602                         continue;
2603                 }
2604
2605                 if (port_is_bonding_slave(pi)) {
2606                         printf("Please remove port %d from bonded device.\n", pi);
2607                         continue;
2608                 }
2609
2610                 port = &ports[pi];
2611                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2612                                                 RTE_PORT_HANDLING) == 0)
2613                         continue;
2614
2615                 rte_eth_dev_stop(pi);
2616
2617                 if (rte_atomic16_cmpset(&(port->port_status),
2618                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2619                         printf("Port %d can not be set into stopped\n", pi);
2620                 need_check_link_status = 1;
2621         }
2622         if (need_check_link_status && !no_link_check)
2623                 check_all_ports_link_status(RTE_PORT_ALL);
2624
2625         printf("Done\n");
2626 }
2627
2628 static void
2629 remove_invalid_ports_in(portid_t *array, portid_t *total)
2630 {
2631         portid_t i;
2632         portid_t new_total = 0;
2633
2634         for (i = 0; i < *total; i++)
2635                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2636                         array[new_total] = array[i];
2637                         new_total++;
2638                 }
2639         *total = new_total;
2640 }
2641
2642 static void
2643 remove_invalid_ports(void)
2644 {
2645         remove_invalid_ports_in(ports_ids, &nb_ports);
2646         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2647         nb_cfg_ports = nb_fwd_ports;
2648 }
2649
2650 void
2651 close_port(portid_t pid)
2652 {
2653         portid_t pi;
2654         struct rte_port *port;
2655
2656         if (port_id_is_invalid(pid, ENABLED_WARN))
2657                 return;
2658
2659         printf("Closing ports...\n");
2660
2661         RTE_ETH_FOREACH_DEV(pi) {
2662                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2663                         continue;
2664
2665                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2666                         printf("Please remove port %d from forwarding configuration.\n", pi);
2667                         continue;
2668                 }
2669
2670                 if (port_is_bonding_slave(pi)) {
2671                         printf("Please remove port %d from bonded device.\n", pi);
2672                         continue;
2673                 }
2674
2675                 port = &ports[pi];
2676                 if (rte_atomic16_cmpset(&(port->port_status),
2677                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2678                         printf("Port %d is already closed\n", pi);
2679                         continue;
2680                 }
2681
2682                 if (rte_atomic16_cmpset(&(port->port_status),
2683                         RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) {
2684                         printf("Port %d is now not stopped\n", pi);
2685                         continue;
2686                 }
2687
2688                 if (port->flow_list)
2689                         port_flow_flush(pi);
2690                 rte_eth_dev_close(pi);
2691
2692                 remove_invalid_ports();
2693
2694                 if (rte_atomic16_cmpset(&(port->port_status),
2695                         RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
2696                         printf("Port %d cannot be set to closed\n", pi);
2697         }
2698
2699         printf("Done\n");
2700 }
2701
2702 void
2703 reset_port(portid_t pid)
2704 {
2705         int diag;
2706         portid_t pi;
2707         struct rte_port *port;
2708
2709         if (port_id_is_invalid(pid, ENABLED_WARN))
2710                 return;
2711
2712         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2713                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2714                 printf("Can not reset port(s), please stop port(s) first.\n");
2715                 return;
2716         }
2717
2718         printf("Resetting ports...\n");
2719
2720         RTE_ETH_FOREACH_DEV(pi) {
2721                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2722                         continue;
2723
2724                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2725                         printf("Please remove port %d from forwarding "
2726                                "configuration.\n", pi);
2727                         continue;
2728                 }
2729
2730                 if (port_is_bonding_slave(pi)) {
2731                         printf("Please remove port %d from bonded device.\n",
2732                                pi);
2733                         continue;
2734                 }
2735
2736                 diag = rte_eth_dev_reset(pi);
2737                 if (diag == 0) {
2738                         port = &ports[pi];
2739                         port->need_reconfig = 1;
2740                         port->need_reconfig_queues = 1;
2741                 } else {
2742                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2743                 }
2744         }
2745
2746         printf("Done\n");
2747 }
2748
2749 void
2750 attach_port(char *identifier)
2751 {
2752         portid_t pi;
2753         struct rte_dev_iterator iterator;
2754
2755         printf("Attaching a new port...\n");
2756
2757         if (identifier == NULL) {
2758                 printf("Invalid parameters are specified\n");
2759                 return;
2760         }
2761
2762         if (rte_dev_probe(identifier) < 0) {
2763                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2764                 return;
2765         }
2766
2767         /* first attach mode: event */
2768         if (setup_on_probe_event) {
2769                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2770                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2771                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2772                                         ports[pi].need_setup != 0)
2773                                 setup_attached_port(pi);
2774                 return;
2775         }
2776
2777         /* second attach mode: iterator */
2778         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2779                 /* setup ports matching the devargs used for probing */
2780                 if (port_is_forwarding(pi))
2781                         continue; /* port was already attached before */
2782                 setup_attached_port(pi);
2783         }
2784 }
2785
2786 static void
2787 setup_attached_port(portid_t pi)
2788 {
2789         unsigned int socket_id;
2790         int ret;
2791
2792         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2793         /* if socket_id is invalid, set to the first available socket. */
2794         if (check_socket_id(socket_id) < 0)
2795                 socket_id = socket_ids[0];
2796         reconfig(pi, socket_id);
2797         ret = rte_eth_promiscuous_enable(pi);
2798         if (ret != 0)
2799                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2800                         pi, rte_strerror(-ret));
2801
2802         ports_ids[nb_ports++] = pi;
2803         fwd_ports_ids[nb_fwd_ports++] = pi;
2804         nb_cfg_ports = nb_fwd_ports;
2805         ports[pi].need_setup = 0;
2806         ports[pi].port_status = RTE_PORT_STOPPED;
2807
2808         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2809         printf("Done\n");
2810 }
2811
2812 static void
2813 detach_device(struct rte_device *dev)
2814 {
2815         portid_t sibling;
2816
2817         if (dev == NULL) {
2818                 printf("Device already removed\n");
2819                 return;
2820         }
2821
2822         printf("Removing a device...\n");
2823
2824         if (rte_dev_remove(dev) < 0) {
2825                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2826                 return;
2827         }
2828         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2829                 /* reset mapping between old ports and removed device */
2830                 rte_eth_devices[sibling].device = NULL;
2831                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2832                         /* sibling ports are forced to be closed */
2833                         ports[sibling].port_status = RTE_PORT_CLOSED;
2834                         printf("Port %u is closed\n", sibling);
2835                 }
2836         }
2837
2838         remove_invalid_ports();
2839
2840         printf("Device is detached\n");
2841         printf("Now total ports is %d\n", nb_ports);
2842         printf("Done\n");
2843         return;
2844 }
2845
2846 void
2847 detach_port_device(portid_t port_id)
2848 {
2849         if (port_id_is_invalid(port_id, ENABLED_WARN))
2850                 return;
2851
2852         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2853                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2854                         printf("Port not stopped\n");
2855                         return;
2856                 }
2857                 printf("Port was not closed\n");
2858                 if (ports[port_id].flow_list)
2859                         port_flow_flush(port_id);
2860         }
2861
2862         detach_device(rte_eth_devices[port_id].device);
2863 }
2864
2865 void
2866 detach_devargs(char *identifier)
2867 {
2868         struct rte_dev_iterator iterator;
2869         struct rte_devargs da;
2870         portid_t port_id;
2871
2872         printf("Removing a device...\n");
2873
2874         memset(&da, 0, sizeof(da));
2875         if (rte_devargs_parsef(&da, "%s", identifier)) {
2876                 printf("cannot parse identifier\n");
2877                 if (da.args)
2878                         free(da.args);
2879                 return;
2880         }
2881
2882         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
2883                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2884                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2885                                 printf("Port %u not stopped\n", port_id);
2886                                 rte_eth_iterator_cleanup(&iterator);
2887                                 return;
2888                         }
2889
2890                         /* sibling ports are forced to be closed */
2891                         if (ports[port_id].flow_list)
2892                                 port_flow_flush(port_id);
2893                         ports[port_id].port_status = RTE_PORT_CLOSED;
2894                         printf("Port %u is now closed\n", port_id);
2895                 }
2896         }
2897
2898         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
2899                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
2900                             da.name, da.bus->name);
2901                 return;
2902         }
2903
2904         remove_invalid_ports();
2905
2906         printf("Device %s is detached\n", identifier);
2907         printf("Now total ports is %d\n", nb_ports);
2908         printf("Done\n");
2909 }
2910
2911 void
2912 pmd_test_exit(void)
2913 {
2914         portid_t pt_id;
2915         int ret;
2916         int i;
2917
2918         if (test_done == 0)
2919                 stop_packet_forwarding();
2920
2921         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2922                 if (mempools[i]) {
2923                         if (mp_alloc_type == MP_ALLOC_ANON)
2924                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
2925                                                      NULL);
2926                 }
2927         }
2928         if (ports != NULL) {
2929                 no_link_check = 1;
2930                 RTE_ETH_FOREACH_DEV(pt_id) {
2931                         printf("\nStopping port %d...\n", pt_id);
2932                         fflush(stdout);
2933                         stop_port(pt_id);
2934                 }
2935                 RTE_ETH_FOREACH_DEV(pt_id) {
2936                         printf("\nShutting down port %d...\n", pt_id);
2937                         fflush(stdout);
2938                         close_port(pt_id);
2939                 }
2940         }
2941
2942         if (hot_plug) {
2943                 ret = rte_dev_event_monitor_stop();
2944                 if (ret) {
2945                         RTE_LOG(ERR, EAL,
2946                                 "fail to stop device event monitor.");
2947                         return;
2948                 }
2949
2950                 ret = rte_dev_event_callback_unregister(NULL,
2951                         dev_event_callback, NULL);
2952                 if (ret < 0) {
2953                         RTE_LOG(ERR, EAL,
2954                                 "fail to unregister device event callback.\n");
2955                         return;
2956                 }
2957
2958                 ret = rte_dev_hotplug_handle_disable();
2959                 if (ret) {
2960                         RTE_LOG(ERR, EAL,
2961                                 "fail to disable hotplug handling.\n");
2962                         return;
2963                 }
2964         }
2965         for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) {
2966                 if (mempools[i])
2967                         rte_mempool_free(mempools[i]);
2968         }
2969
2970         printf("\nBye...\n");
2971 }
2972
2973 typedef void (*cmd_func_t)(void);
2974 struct pmd_test_command {
2975         const char *cmd_name;
2976         cmd_func_t cmd_func;
2977 };
2978
2979 /* Check the link status of all ports in up to 9s, and print them finally */
2980 static void
2981 check_all_ports_link_status(uint32_t port_mask)
2982 {
2983 #define CHECK_INTERVAL 100 /* 100ms */
2984 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2985         portid_t portid;
2986         uint8_t count, all_ports_up, print_flag = 0;
2987         struct rte_eth_link link;
2988         int ret;
2989
2990         printf("Checking link statuses...\n");
2991         fflush(stdout);
2992         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2993                 all_ports_up = 1;
2994                 RTE_ETH_FOREACH_DEV(portid) {
2995                         if ((port_mask & (1 << portid)) == 0)
2996                                 continue;
2997                         memset(&link, 0, sizeof(link));
2998                         ret = rte_eth_link_get_nowait(portid, &link);
2999                         if (ret < 0) {
3000                                 all_ports_up = 0;
3001                                 if (print_flag == 1)
3002                                         printf("Port %u link get failed: %s\n",
3003                                                 portid, rte_strerror(-ret));
3004                                 continue;
3005                         }
3006                         /* print link status if flag set */
3007                         if (print_flag == 1) {
3008                                 if (link.link_status)
3009                                         printf(
3010                                         "Port%d Link Up. speed %u Mbps- %s\n",
3011                                         portid, link.link_speed,
3012                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
3013                                         ("full-duplex") : ("half-duplex"));
3014                                 else
3015                                         printf("Port %d Link Down\n", portid);
3016                                 continue;
3017                         }
3018                         /* clear all_ports_up flag if any link down */
3019                         if (link.link_status == ETH_LINK_DOWN) {
3020                                 all_ports_up = 0;
3021                                 break;
3022                         }
3023                 }
3024                 /* after finally printing all link status, get out */
3025                 if (print_flag == 1)
3026                         break;
3027
3028                 if (all_ports_up == 0) {
3029                         fflush(stdout);
3030                         rte_delay_ms(CHECK_INTERVAL);
3031                 }
3032
3033                 /* set the print_flag if all ports up or timeout */
3034                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3035                         print_flag = 1;
3036                 }
3037
3038                 if (lsc_interrupt)
3039                         break;
3040         }
3041 }
3042
3043 /*
3044  * This callback is for remove a port for a device. It has limitation because
3045  * it is not for multiple port removal for a device.
3046  * TODO: the device detach invoke will plan to be removed from user side to
3047  * eal. And convert all PMDs to free port resources on ether device closing.
3048  */
3049 static void
3050 rmv_port_callback(void *arg)
3051 {
3052         int need_to_start = 0;
3053         int org_no_link_check = no_link_check;
3054         portid_t port_id = (intptr_t)arg;
3055         struct rte_device *dev;
3056
3057         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3058
3059         if (!test_done && port_is_forwarding(port_id)) {
3060                 need_to_start = 1;
3061                 stop_packet_forwarding();
3062         }
3063         no_link_check = 1;
3064         stop_port(port_id);
3065         no_link_check = org_no_link_check;
3066
3067         /* Save rte_device pointer before closing ethdev port */
3068         dev = rte_eth_devices[port_id].device;
3069         close_port(port_id);
3070         detach_device(dev); /* might be already removed or have more ports */
3071
3072         if (need_to_start)
3073                 start_packet_forwarding(0);
3074 }
3075
3076 /* This function is used by the interrupt thread */
3077 static int
3078 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3079                   void *ret_param)
3080 {
3081         RTE_SET_USED(param);
3082         RTE_SET_USED(ret_param);
3083
3084         if (type >= RTE_ETH_EVENT_MAX) {
3085                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3086                         port_id, __func__, type);
3087                 fflush(stderr);
3088         } else if (event_print_mask & (UINT32_C(1) << type)) {
3089                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3090                         eth_event_desc[type]);
3091                 fflush(stdout);
3092         }
3093
3094         switch (type) {
3095         case RTE_ETH_EVENT_NEW:
3096                 ports[port_id].need_setup = 1;
3097                 ports[port_id].port_status = RTE_PORT_HANDLING;
3098                 break;
3099         case RTE_ETH_EVENT_INTR_RMV:
3100                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3101                         break;
3102                 if (rte_eal_alarm_set(100000,
3103                                 rmv_port_callback, (void *)(intptr_t)port_id))
3104                         fprintf(stderr, "Could not set up deferred device removal\n");
3105                 break;
3106         default:
3107                 break;
3108         }
3109         return 0;
3110 }
3111
3112 static int
3113 register_eth_event_callback(void)
3114 {
3115         int ret;
3116         enum rte_eth_event_type event;
3117
3118         for (event = RTE_ETH_EVENT_UNKNOWN;
3119                         event < RTE_ETH_EVENT_MAX; event++) {
3120                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3121                                 event,
3122                                 eth_event_callback,
3123                                 NULL);
3124                 if (ret != 0) {
3125                         TESTPMD_LOG(ERR, "Failed to register callback for "
3126                                         "%s event\n", eth_event_desc[event]);
3127                         return -1;
3128                 }
3129         }
3130
3131         return 0;
3132 }
3133
3134 /* This function is used by the interrupt thread */
3135 static void
3136 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3137                              __rte_unused void *arg)
3138 {
3139         uint16_t port_id;
3140         int ret;
3141
3142         if (type >= RTE_DEV_EVENT_MAX) {
3143                 fprintf(stderr, "%s called upon invalid event %d\n",
3144                         __func__, type);
3145                 fflush(stderr);
3146         }
3147
3148         switch (type) {
3149         case RTE_DEV_EVENT_REMOVE:
3150                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3151                         device_name);
3152                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3153                 if (ret) {
3154                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3155                                 device_name);
3156                         return;
3157                 }
3158                 /*
3159                  * Because the user's callback is invoked in eal interrupt
3160                  * callback, the interrupt callback need to be finished before
3161                  * it can be unregistered when detaching device. So finish
3162                  * callback soon and use a deferred removal to detach device
3163                  * is need. It is a workaround, once the device detaching be
3164                  * moved into the eal in the future, the deferred removal could
3165                  * be deleted.
3166                  */
3167                 if (rte_eal_alarm_set(100000,
3168                                 rmv_port_callback, (void *)(intptr_t)port_id))
3169                         RTE_LOG(ERR, EAL,
3170                                 "Could not set up deferred device removal\n");
3171                 break;
3172         case RTE_DEV_EVENT_ADD:
3173                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3174                         device_name);
3175                 /* TODO: After finish kernel driver binding,
3176                  * begin to attach port.
3177                  */
3178                 break;
3179         default:
3180                 break;
3181         }
3182 }
3183
3184 static int
3185 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3186 {
3187         uint16_t i;
3188         int diag;
3189         uint8_t mapping_found = 0;
3190
3191         for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3192                 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3193                                 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3194                         diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3195                                         tx_queue_stats_mappings[i].queue_id,
3196                                         tx_queue_stats_mappings[i].stats_counter_id);
3197                         if (diag != 0)
3198                                 return diag;
3199                         mapping_found = 1;
3200                 }
3201         }
3202         if (mapping_found)
3203                 port->tx_queue_stats_mapping_enabled = 1;
3204         return 0;
3205 }
3206
3207 static int
3208 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3209 {
3210         uint16_t i;
3211         int diag;
3212         uint8_t mapping_found = 0;
3213
3214         for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3215                 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3216                                 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3217                         diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3218                                         rx_queue_stats_mappings[i].queue_id,
3219                                         rx_queue_stats_mappings[i].stats_counter_id);
3220                         if (diag != 0)
3221                                 return diag;
3222                         mapping_found = 1;
3223                 }
3224         }
3225         if (mapping_found)
3226                 port->rx_queue_stats_mapping_enabled = 1;
3227         return 0;
3228 }
3229
3230 static void
3231 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3232 {
3233         int diag = 0;
3234
3235         diag = set_tx_queue_stats_mapping_registers(pi, port);
3236         if (diag != 0) {
3237                 if (diag == -ENOTSUP) {
3238                         port->tx_queue_stats_mapping_enabled = 0;
3239                         printf("TX queue stats mapping not supported port id=%d\n", pi);
3240                 }
3241                 else
3242                         rte_exit(EXIT_FAILURE,
3243                                         "set_tx_queue_stats_mapping_registers "
3244                                         "failed for port id=%d diag=%d\n",
3245                                         pi, diag);
3246         }
3247
3248         diag = set_rx_queue_stats_mapping_registers(pi, port);
3249         if (diag != 0) {
3250                 if (diag == -ENOTSUP) {
3251                         port->rx_queue_stats_mapping_enabled = 0;
3252                         printf("RX queue stats mapping not supported port id=%d\n", pi);
3253                 }
3254                 else
3255                         rte_exit(EXIT_FAILURE,
3256                                         "set_rx_queue_stats_mapping_registers "
3257                                         "failed for port id=%d diag=%d\n",
3258                                         pi, diag);
3259         }
3260 }
3261
3262 static void
3263 rxtx_port_config(struct rte_port *port)
3264 {
3265         uint16_t qid;
3266         uint64_t offloads;
3267
3268         for (qid = 0; qid < nb_rxq; qid++) {
3269                 offloads = port->rx_conf[qid].offloads;
3270                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3271                 if (offloads != 0)
3272                         port->rx_conf[qid].offloads = offloads;
3273
3274                 /* Check if any Rx parameters have been passed */
3275                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3276                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3277
3278                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3279                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3280
3281                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3282                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3283
3284                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3285                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3286
3287                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3288                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3289
3290                 port->nb_rx_desc[qid] = nb_rxd;
3291         }
3292
3293         for (qid = 0; qid < nb_txq; qid++) {
3294                 offloads = port->tx_conf[qid].offloads;
3295                 port->tx_conf[qid] = port->dev_info.default_txconf;
3296                 if (offloads != 0)
3297                         port->tx_conf[qid].offloads = offloads;
3298
3299                 /* Check if any Tx parameters have been passed */
3300                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3301                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3302
3303                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3304                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3305
3306                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3307                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3308
3309                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3310                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3311
3312                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3313                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3314
3315                 port->nb_tx_desc[qid] = nb_txd;
3316         }
3317 }
3318
3319 void
3320 init_port_config(void)
3321 {
3322         portid_t pid;
3323         struct rte_port *port;
3324         int ret;
3325
3326         RTE_ETH_FOREACH_DEV(pid) {
3327                 port = &ports[pid];
3328                 port->dev_conf.fdir_conf = fdir_conf;
3329
3330                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3331                 if (ret != 0)
3332                         return;
3333
3334                 if (nb_rxq > 1) {
3335                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3336                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3337                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3338                 } else {
3339                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3340                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3341                 }
3342
3343                 if (port->dcb_flag == 0) {
3344                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3345                                 port->dev_conf.rxmode.mq_mode =
3346                                         (enum rte_eth_rx_mq_mode)
3347                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3348                         else
3349                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3350                 }
3351
3352                 rxtx_port_config(port);
3353
3354                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3355                 if (ret != 0)
3356                         return;
3357
3358                 map_port_queue_stats_mapping_registers(pid, port);
3359 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS
3360                 rte_pmd_ixgbe_bypass_init(pid);
3361 #endif
3362
3363                 if (lsc_interrupt &&
3364                     (rte_eth_devices[pid].data->dev_flags &
3365                      RTE_ETH_DEV_INTR_LSC))
3366                         port->dev_conf.intr_conf.lsc = 1;
3367                 if (rmv_interrupt &&
3368                     (rte_eth_devices[pid].data->dev_flags &
3369                      RTE_ETH_DEV_INTR_RMV))
3370                         port->dev_conf.intr_conf.rmv = 1;
3371         }
3372 }
3373
3374 void set_port_slave_flag(portid_t slave_pid)
3375 {
3376         struct rte_port *port;
3377
3378         port = &ports[slave_pid];
3379         port->slave_flag = 1;
3380 }
3381
3382 void clear_port_slave_flag(portid_t slave_pid)
3383 {
3384         struct rte_port *port;
3385
3386         port = &ports[slave_pid];
3387         port->slave_flag = 0;
3388 }
3389
3390 uint8_t port_is_bonding_slave(portid_t slave_pid)
3391 {
3392         struct rte_port *port;
3393
3394         port = &ports[slave_pid];
3395         if ((rte_eth_devices[slave_pid].data->dev_flags &
3396             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3397                 return 1;
3398         return 0;
3399 }
3400
3401 const uint16_t vlan_tags[] = {
3402                 0,  1,  2,  3,  4,  5,  6,  7,
3403                 8,  9, 10, 11,  12, 13, 14, 15,
3404                 16, 17, 18, 19, 20, 21, 22, 23,
3405                 24, 25, 26, 27, 28, 29, 30, 31
3406 };
3407
3408 static  int
3409 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3410                  enum dcb_mode_enable dcb_mode,
3411                  enum rte_eth_nb_tcs num_tcs,
3412                  uint8_t pfc_en)
3413 {
3414         uint8_t i;
3415         int32_t rc;
3416         struct rte_eth_rss_conf rss_conf;
3417
3418         /*
3419          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3420          * given above, and the number of traffic classes available for use.
3421          */
3422         if (dcb_mode == DCB_VT_ENABLED) {
3423                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3424                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3425                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3426                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3427
3428                 /* VMDQ+DCB RX and TX configurations */
3429                 vmdq_rx_conf->enable_default_pool = 0;
3430                 vmdq_rx_conf->default_pool = 0;
3431                 vmdq_rx_conf->nb_queue_pools =
3432                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3433                 vmdq_tx_conf->nb_queue_pools =
3434                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3435
3436                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3437                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3438                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3439                         vmdq_rx_conf->pool_map[i].pools =
3440                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3441                 }
3442                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3443                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3444                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3445                 }
3446
3447                 /* set DCB mode of RX and TX of multiple queues */
3448                 eth_conf->rxmode.mq_mode =
3449                                 (enum rte_eth_rx_mq_mode)
3450                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3451                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3452         } else {
3453                 struct rte_eth_dcb_rx_conf *rx_conf =
3454                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3455                 struct rte_eth_dcb_tx_conf *tx_conf =
3456                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3457
3458                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3459
3460                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3461                 if (rc != 0)
3462                         return rc;
3463
3464                 rx_conf->nb_tcs = num_tcs;
3465                 tx_conf->nb_tcs = num_tcs;
3466
3467                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3468                         rx_conf->dcb_tc[i] = i % num_tcs;
3469                         tx_conf->dcb_tc[i] = i % num_tcs;
3470                 }
3471
3472                 eth_conf->rxmode.mq_mode =
3473                                 (enum rte_eth_rx_mq_mode)
3474                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3475                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3476                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3477         }
3478
3479         if (pfc_en)
3480                 eth_conf->dcb_capability_en =
3481                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3482         else
3483                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3484
3485         return 0;
3486 }
3487
3488 int
3489 init_port_dcb_config(portid_t pid,
3490                      enum dcb_mode_enable dcb_mode,
3491                      enum rte_eth_nb_tcs num_tcs,
3492                      uint8_t pfc_en)
3493 {
3494         struct rte_eth_conf port_conf;
3495         struct rte_port *rte_port;
3496         int retval;
3497         uint16_t i;
3498
3499         rte_port = &ports[pid];
3500
3501         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3502         /* Enter DCB configuration status */
3503         dcb_config = 1;
3504
3505         port_conf.rxmode = rte_port->dev_conf.rxmode;
3506         port_conf.txmode = rte_port->dev_conf.txmode;
3507
3508         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3509         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3510         if (retval < 0)
3511                 return retval;
3512         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3513
3514         /* re-configure the device . */
3515         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3516         if (retval < 0)
3517                 return retval;
3518
3519         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3520         if (retval != 0)
3521                 return retval;
3522
3523         /* If dev_info.vmdq_pool_base is greater than 0,
3524          * the queue id of vmdq pools is started after pf queues.
3525          */
3526         if (dcb_mode == DCB_VT_ENABLED &&
3527             rte_port->dev_info.vmdq_pool_base > 0) {
3528                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3529                         " for port %d.", pid);
3530                 return -1;
3531         }
3532
3533         /* Assume the ports in testpmd have the same dcb capability
3534          * and has the same number of rxq and txq in dcb mode
3535          */
3536         if (dcb_mode == DCB_VT_ENABLED) {
3537                 if (rte_port->dev_info.max_vfs > 0) {
3538                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3539                         nb_txq = rte_port->dev_info.nb_tx_queues;
3540                 } else {
3541                         nb_rxq = rte_port->dev_info.max_rx_queues;
3542                         nb_txq = rte_port->dev_info.max_tx_queues;
3543                 }
3544         } else {
3545                 /*if vt is disabled, use all pf queues */
3546                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3547                         nb_rxq = rte_port->dev_info.max_rx_queues;
3548                         nb_txq = rte_port->dev_info.max_tx_queues;
3549                 } else {
3550                         nb_rxq = (queueid_t)num_tcs;
3551                         nb_txq = (queueid_t)num_tcs;
3552
3553                 }
3554         }
3555         rx_free_thresh = 64;
3556
3557         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3558
3559         rxtx_port_config(rte_port);
3560         /* VLAN filter */
3561         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3562         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3563                 rx_vft_set(pid, vlan_tags[i], 1);
3564
3565         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3566         if (retval != 0)
3567                 return retval;
3568
3569         map_port_queue_stats_mapping_registers(pid, rte_port);
3570
3571         rte_port->dcb_flag = 1;
3572
3573         return 0;
3574 }
3575
3576 static void
3577 init_port(void)
3578 {
3579         /* Configuration of Ethernet ports. */
3580         ports = rte_zmalloc("testpmd: ports",
3581                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3582                             RTE_CACHE_LINE_SIZE);
3583         if (ports == NULL) {
3584                 rte_exit(EXIT_FAILURE,
3585                                 "rte_zmalloc(%d struct rte_port) failed\n",
3586                                 RTE_MAX_ETHPORTS);
3587         }
3588
3589         /* Initialize ports NUMA structures */
3590         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3591         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3592         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3593 }
3594
3595 static void
3596 force_quit(void)
3597 {
3598         pmd_test_exit();
3599         prompt_exit();
3600 }
3601
3602 static void
3603 print_stats(void)
3604 {
3605         uint8_t i;
3606         const char clr[] = { 27, '[', '2', 'J', '\0' };
3607         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3608
3609         /* Clear screen and move to top left */
3610         printf("%s%s", clr, top_left);
3611
3612         printf("\nPort statistics ====================================");
3613         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3614                 nic_stats_display(fwd_ports_ids[i]);
3615
3616         fflush(stdout);
3617 }
3618
3619 static void
3620 signal_handler(int signum)
3621 {
3622         if (signum == SIGINT || signum == SIGTERM) {
3623                 printf("\nSignal %d received, preparing to exit...\n",
3624                                 signum);
3625 #ifdef RTE_LIBRTE_PDUMP
3626                 /* uninitialize packet capture framework */
3627                 rte_pdump_uninit();
3628 #endif
3629 #ifdef RTE_LIBRTE_LATENCY_STATS
3630                 if (latencystats_enabled != 0)
3631                         rte_latencystats_uninit();
3632 #endif
3633                 force_quit();
3634                 /* Set flag to indicate the force termination. */
3635                 f_quit = 1;
3636                 /* exit with the expected status */
3637                 signal(signum, SIG_DFL);
3638                 kill(getpid(), signum);
3639         }
3640 }
3641
3642 int
3643 main(int argc, char** argv)
3644 {
3645         int diag;
3646         portid_t port_id;
3647         uint16_t count;
3648         int ret;
3649
3650         signal(SIGINT, signal_handler);
3651         signal(SIGTERM, signal_handler);
3652
3653         testpmd_logtype = rte_log_register("testpmd");
3654         if (testpmd_logtype < 0)
3655                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3656         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3657
3658         diag = rte_eal_init(argc, argv);
3659         if (diag < 0)
3660                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3661                          rte_strerror(rte_errno));
3662
3663         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3664                 rte_exit(EXIT_FAILURE,
3665                          "Secondary process type not supported.\n");
3666
3667         ret = register_eth_event_callback();
3668         if (ret != 0)
3669                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3670
3671 #ifdef RTE_LIBRTE_PDUMP
3672         /* initialize packet capture framework */
3673         rte_pdump_init();
3674 #endif
3675
3676         count = 0;
3677         RTE_ETH_FOREACH_DEV(port_id) {
3678                 ports_ids[count] = port_id;
3679                 count++;
3680         }
3681         nb_ports = (portid_t) count;
3682         if (nb_ports == 0)
3683                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3684
3685         /* allocate port structures, and init them */
3686         init_port();
3687
3688         set_def_fwd_config();
3689         if (nb_lcores == 0)
3690                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3691                          "Check the core mask argument\n");
3692
3693         /* Bitrate/latency stats disabled by default */
3694 #ifdef RTE_LIBRTE_BITRATE
3695         bitrate_enabled = 0;
3696 #endif
3697 #ifdef RTE_LIBRTE_LATENCY_STATS
3698         latencystats_enabled = 0;
3699 #endif
3700
3701         /* on FreeBSD, mlockall() is disabled by default */
3702 #ifdef RTE_EXEC_ENV_FREEBSD
3703         do_mlockall = 0;
3704 #else
3705         do_mlockall = 1;
3706 #endif
3707
3708         argc -= diag;
3709         argv += diag;
3710         if (argc > 1)
3711                 launch_args_parse(argc, argv);
3712
3713         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3714                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3715                         strerror(errno));
3716         }
3717
3718         if (tx_first && interactive)
3719                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3720                                 "interactive mode.\n");
3721
3722         if (tx_first && lsc_interrupt) {
3723                 printf("Warning: lsc_interrupt needs to be off when "
3724                                 " using tx_first. Disabling.\n");
3725                 lsc_interrupt = 0;
3726         }
3727
3728         if (!nb_rxq && !nb_txq)
3729                 printf("Warning: Either rx or tx queues should be non-zero\n");
3730
3731         if (nb_rxq > 1 && nb_rxq > nb_txq)
3732                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3733                        "but nb_txq=%d will prevent to fully test it.\n",
3734                        nb_rxq, nb_txq);
3735
3736         init_config();
3737
3738         if (hot_plug) {
3739                 ret = rte_dev_hotplug_handle_enable();
3740                 if (ret) {
3741                         RTE_LOG(ERR, EAL,
3742                                 "fail to enable hotplug handling.");
3743                         return -1;
3744                 }
3745
3746                 ret = rte_dev_event_monitor_start();
3747                 if (ret) {
3748                         RTE_LOG(ERR, EAL,
3749                                 "fail to start device event monitoring.");
3750                         return -1;
3751                 }
3752
3753                 ret = rte_dev_event_callback_register(NULL,
3754                         dev_event_callback, NULL);
3755                 if (ret) {
3756                         RTE_LOG(ERR, EAL,
3757                                 "fail  to register device event callback\n");
3758                         return -1;
3759                 }
3760         }
3761
3762         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3763                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3764
3765         /* set all ports to promiscuous mode by default */
3766         RTE_ETH_FOREACH_DEV(port_id) {
3767                 ret = rte_eth_promiscuous_enable(port_id);
3768                 if (ret != 0)
3769                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3770                                 port_id, rte_strerror(-ret));
3771         }
3772
3773         /* Init metrics library */
3774         rte_metrics_init(rte_socket_id());
3775
3776 #ifdef RTE_LIBRTE_LATENCY_STATS
3777         if (latencystats_enabled != 0) {
3778                 int ret = rte_latencystats_init(1, NULL);
3779                 if (ret)
3780                         printf("Warning: latencystats init()"
3781                                 " returned error %d\n", ret);
3782                 printf("Latencystats running on lcore %d\n",
3783                         latencystats_lcore_id);
3784         }
3785 #endif
3786
3787         /* Setup bitrate stats */
3788 #ifdef RTE_LIBRTE_BITRATE
3789         if (bitrate_enabled != 0) {
3790                 bitrate_data = rte_stats_bitrate_create();
3791                 if (bitrate_data == NULL)
3792                         rte_exit(EXIT_FAILURE,
3793                                 "Could not allocate bitrate data.\n");
3794                 rte_stats_bitrate_reg(bitrate_data);
3795         }
3796 #endif
3797
3798 #ifdef RTE_LIBRTE_CMDLINE
3799         if (strlen(cmdline_filename) != 0)
3800                 cmdline_read_from_file(cmdline_filename);
3801
3802         if (interactive == 1) {
3803                 if (auto_start) {
3804                         printf("Start automatic packet forwarding\n");
3805                         start_packet_forwarding(0);
3806                 }
3807                 prompt();
3808                 pmd_test_exit();
3809         } else
3810 #endif
3811         {
3812                 char c;
3813                 int rc;
3814
3815                 f_quit = 0;
3816
3817                 printf("No commandline core given, start packet forwarding\n");
3818                 start_packet_forwarding(tx_first);
3819                 if (stats_period != 0) {
3820                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3821                         uint64_t timer_period;
3822
3823                         /* Convert to number of cycles */
3824                         timer_period = stats_period * rte_get_timer_hz();
3825
3826                         while (f_quit == 0) {
3827                                 cur_time = rte_get_timer_cycles();
3828                                 diff_time += cur_time - prev_time;
3829
3830                                 if (diff_time >= timer_period) {
3831                                         print_stats();
3832                                         /* Reset the timer */
3833                                         diff_time = 0;
3834                                 }
3835                                 /* Sleep to avoid unnecessary checks */
3836                                 prev_time = cur_time;
3837                                 sleep(1);
3838                         }
3839                 }
3840
3841                 printf("Press enter to exit\n");
3842                 rc = read(0, &c, 1);
3843                 pmd_test_exit();
3844                 if (rc < 0)
3845                         return 1;
3846         }
3847
3848         ret = rte_eal_cleanup();
3849         if (ret != 0)
3850                 rte_exit(EXIT_FAILURE,
3851                          "EAL cleanup failed: %s\n", strerror(-ret));
3852
3853         return EXIT_SUCCESS;
3854 }