app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_NET_IXGBE
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIB_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIB_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIB_LATENCYSTATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use main core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 219 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 220
 221 /*
 222  * Configuration of packet segments used by the "txonly" processing engine.
 223  */
 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 226         TXONLY_DEF_PACKET_LEN,
 227 };
 228 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 229
 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 231 /**< Split policy for packets to TX. */
 232
 233 uint8_t txonly_multi_flow;
 234 /**< Whether multiple flows are generated in TXONLY mode. */
 235
 236 uint32_t tx_pkt_times_inter;
 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 238
 239 uint32_t tx_pkt_times_intra;
 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 241
 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 245
 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 247 uint8_t dcb_config = 0;
 248
 249 /* Whether the dcb is in testing status */
 250 uint8_t dcb_test = 0;
 251
 252 /*
 253  * Configurable number of RX/TX queues.
 254  */
 255 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 256 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 257 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 258
 259 /*
 260  * Configurable number of RX/TX ring descriptors.
 261  * Defaults are supplied by drivers via ethdev.
 262  */
 263 #define RTE_TEST_RX_DESC_DEFAULT 0
 264 #define RTE_TEST_TX_DESC_DEFAULT 0
 265 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 266 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 267
 268 #define RTE_PMD_PARAM_UNSET -1
 269 /*
 270  * Configurable values of RX and TX ring threshold registers.
 271  */
 272
 273 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 274 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 275 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 276
 277 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 279 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 280
 281 /*
 282  * Configurable value of RX free threshold.
 283  */
 284 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 285
 286 /*
 287  * Configurable value of RX drop enable.
 288  */
 289 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 290
 291 /*
 292  * Configurable value of TX free threshold.
 293  */
 294 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 295
 296 /*
 297  * Configurable value of TX RS bit threshold.
 298  */
 299 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 300
 301 /*
 302  * Configurable value of buffered packets before sending.
 303  */
 304 uint16_t noisy_tx_sw_bufsz;
 305
 306 /*
 307  * Configurable value of packet buffer timeout.
 308  */
 309 uint16_t noisy_tx_sw_buf_flush_time;
 310
 311 /*
 312  * Configurable value for size of VNF internal memory area
 313  * used for simulating noisy neighbour behaviour
 314  */
 315 uint64_t noisy_lkup_mem_sz;
 316
 317 /*
 318  * Configurable value of number of random writes done in
 319  * VNF simulation memory area.
 320  */
 321 uint64_t noisy_lkup_num_writes;
 322
 323 /*
 324  * Configurable value of number of random reads done in
 325  * VNF simulation memory area.
 326  */
 327 uint64_t noisy_lkup_num_reads;
 328
 329 /*
 330  * Configurable value of number of random reads/writes done in
 331  * VNF simulation memory area.
 332  */
 333 uint64_t noisy_lkup_num_reads_writes;
 334
 335 /*
 336  * Receive Side Scaling (RSS) configuration.
 337  */
 338 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 339
 340 /*
 341  * Port topology configuration
 342  */
 343 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 344
 345 /*
 346  * Avoids to flush all the RX streams before starts forwarding.
 347  */
 348 uint8_t no_flush_rx = 0; /* flush by default */
 349
 350 /*
 351  * Flow API isolated mode.
 352  */
 353 uint8_t flow_isolate_all;
 354
 355 /*
 356  * Avoids to check link status when starting/stopping a port.
 357  */
 358 uint8_t no_link_check = 0; /* check by default */
 359
 360 /*
 361  * Don't automatically start all ports in interactive mode.
 362  */
 363 uint8_t no_device_start = 0;
 364
 365 /*
 366  * Enable link status change notification
 367  */
 368 uint8_t lsc_interrupt = 1; /* enabled by default */
 369
 370 /*
 371  * Enable device removal notification.
 372  */
 373 uint8_t rmv_interrupt = 1; /* enabled by default */
 374
 375 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 376
 377 /* After attach, port setup is called on event or by iterator */
 378 bool setup_on_probe_event = true;
 379
 380 /* Clear ptypes on port initialization. */
 381 uint8_t clear_ptypes = true;
 382
 383 /* Hairpin ports configuration mode. */
 384 uint16_t hairpin_mode;
 385
 386 /* Pretty printing of ethdev events */
 387 static const char * const eth_event_desc[] = {
 388         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 389         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 390         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 391         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 392         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 393         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 394         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 395         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 396         [RTE_ETH_EVENT_NEW] = "device probed",
 397         [RTE_ETH_EVENT_DESTROY] = "device released",
 398         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 399         [RTE_ETH_EVENT_MAX] = NULL,
 400 };
 401
 402 /*
 403  * Display or mask ether events
 404  * Default to all events except VF_MBOX
 405  */
 406 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 414 /*
 415  * Decide if all memory are locked for performance.
 416  */
 417 int do_mlockall = 0;
 418
 419 /*
 420  * NIC bypass mode configuration options.
 421  */
 422
 423 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 424 /* The NIC bypass watchdog timeout. */
 425 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 426 #endif
 427
 428
 429 #ifdef RTE_LIB_LATENCYSTATS
 430
 431 /*
 432  * Set when latency stats is enabled in the commandline
 433  */
 434 uint8_t latencystats_enabled;
 435
 436 /*
 437  * Lcore ID to serive latency statistics.
 438  */
 439 lcoreid_t latencystats_lcore_id = -1;
 440
 441 #endif
 442
 443 /*
 444  * Ethernet device configuration.
 445  */
 446 struct rte_eth_rxmode rx_mode = {
 447         /* Default maximum frame length.
 448          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 449          * in init_config().
 450          */
 451         .max_rx_pkt_len = 0,
 452 };
 453
 454 struct rte_eth_txmode tx_mode = {
 455         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 456 };
 457
 458 struct rte_fdir_conf fdir_conf = {
 459         .mode = RTE_FDIR_MODE_NONE,
 460         .pballoc = RTE_FDIR_PBALLOC_64K,
 461         .status = RTE_FDIR_REPORT_STATUS,
 462         .mask = {
 463                 .vlan_tci_mask = 0xFFEF,
 464                 .ipv4_mask     = {
 465                         .src_ip = 0xFFFFFFFF,
 466                         .dst_ip = 0xFFFFFFFF,
 467                 },
 468                 .ipv6_mask     = {
 469                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 470                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 471                 },
 472                 .src_port_mask = 0xFFFF,
 473                 .dst_port_mask = 0xFFFF,
 474                 .mac_addr_byte_mask = 0xFF,
 475                 .tunnel_type_mask = 1,
 476                 .tunnel_id_mask = 0xFFFFFFFF,
 477         },
 478         .drop_queue = 127,
 479 };
 480
 481 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 482
 483 /*
 484  * Display zero values by default for xstats
 485  */
 486 uint8_t xstats_hide_zero;
 487
 488 /*
 489  * Measure of CPU cycles disabled by default
 490  */
 491 uint8_t record_core_cycles;
 492
 493 /*
 494  * Display of RX and TX bursts disabled by default
 495  */
 496 uint8_t record_burst_stats;
 497
 498 unsigned int num_sockets = 0;
 499 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 500
 501 #ifdef RTE_LIB_BITRATESTATS
 502 /* Bitrate statistics */
 503 struct rte_stats_bitrates *bitrate_data;
 504 lcoreid_t bitrate_lcore_id;
 505 uint8_t bitrate_enabled;
 506 #endif
 507
 508 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 509 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 510
 511 /*
 512  * hexadecimal bitmask of RX mq mode can be enabled.
 513  */
 514 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 515
 516 /*
 517  * Used to set forced link speed
 518  */
 519 uint32_t eth_link_speed;
 520
 521 /* Forward function declarations */
 522 static void setup_attached_port(portid_t pi);
 523 static void check_all_ports_link_status(uint32_t port_mask);
 524 static int eth_event_callback(portid_t port_id,
 525                               enum rte_eth_event_type type,
 526                               void *param, void *ret_param);
 527 static void dev_event_callback(const char *device_name,
 528                                 enum rte_dev_event_type type,
 529                                 void *param);
 530
 531 /*
 532  * Check if all the ports are started.
 533  * If yes, return positive value. If not, return zero.
 534  */
 535 static int all_ports_started(void);
 536
 537 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 538 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 539
 540 /* Holds the registered mbuf dynamic flags names. */
 541 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 542
 543 /*
 544  * Helper function to check if socket is already discovered.
 545  * If yes, return positive value. If not, return zero.
 546  */
 547 int
 548 new_socket_id(unsigned int socket_id)
 549 {
 550         unsigned int i;
 551
 552         for (i = 0; i < num_sockets; i++) {
 553                 if (socket_ids[i] == socket_id)
 554                         return 0;
 555         }
 556         return 1;
 557 }
 558
 559 /*
 560  * Setup default configuration.
 561  */
 562 static void
 563 set_default_fwd_lcores_config(void)
 564 {
 565         unsigned int i;
 566         unsigned int nb_lc;
 567         unsigned int sock_num;
 568
 569         nb_lc = 0;
 570         for (i = 0; i < RTE_MAX_LCORE; i++) {
 571                 if (!rte_lcore_is_enabled(i))
 572                         continue;
 573                 sock_num = rte_lcore_to_socket_id(i);
 574                 if (new_socket_id(sock_num)) {
 575                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 576                                 rte_exit(EXIT_FAILURE,
 577                                          "Total sockets greater than %u\n",
 578                                          RTE_MAX_NUMA_NODES);
 579                         }
 580                         socket_ids[num_sockets++] = sock_num;
 581                 }
 582                 if (i == rte_get_main_lcore())
 583                         continue;
 584                 fwd_lcores_cpuids[nb_lc++] = i;
 585         }
 586         nb_lcores = (lcoreid_t) nb_lc;
 587         nb_cfg_lcores = nb_lcores;
 588         nb_fwd_lcores = 1;
 589 }
 590
 591 static void
 592 set_def_peer_eth_addrs(void)
 593 {
 594         portid_t i;
 595
 596         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 597                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 598                 peer_eth_addrs[i].addr_bytes[5] = i;
 599         }
 600 }
 601
 602 static void
 603 set_default_fwd_ports_config(void)
 604 {
 605         portid_t pt_id;
 606         int i = 0;
 607
 608         RTE_ETH_FOREACH_DEV(pt_id) {
 609                 fwd_ports_ids[i++] = pt_id;
 610
 611                 /* Update sockets info according to the attached device */
 612                 int socket_id = rte_eth_dev_socket_id(pt_id);
 613                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 614                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 615                                 rte_exit(EXIT_FAILURE,
 616                                          "Total sockets greater than %u\n",
 617                                          RTE_MAX_NUMA_NODES);
 618                         }
 619                         socket_ids[num_sockets++] = socket_id;
 620                 }
 621         }
 622
 623         nb_cfg_ports = nb_ports;
 624         nb_fwd_ports = nb_ports;
 625 }
 626
 627 void
 628 set_def_fwd_config(void)
 629 {
 630         set_default_fwd_lcores_config();
 631         set_def_peer_eth_addrs();
 632         set_default_fwd_ports_config();
 633 }
 634
 635 /* extremely pessimistic estimation of memory required to create a mempool */
 636 static int
 637 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 638 {
 639         unsigned int n_pages, mbuf_per_pg, leftover;
 640         uint64_t total_mem, mbuf_mem, obj_sz;
 641
 642         /* there is no good way to predict how much space the mempool will
 643          * occupy because it will allocate chunks on the fly, and some of those
 644          * will come from default DPDK memory while some will come from our
 645          * external memory, so just assume 128MB will be enough for everyone.
 646          */
 647         uint64_t hdr_mem = 128 << 20;
 648
 649         /* account for possible non-contiguousness */
 650         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 651         if (obj_sz > pgsz) {
 652                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 653                 return -1;
 654         }
 655
 656         mbuf_per_pg = pgsz / obj_sz;
 657         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 658         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 659
 660         mbuf_mem = n_pages * pgsz;
 661
 662         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 663
 664         if (total_mem > SIZE_MAX) {
 665                 TESTPMD_LOG(ERR, "Memory size too big\n");
 666                 return -1;
 667         }
 668         *out = (size_t)total_mem;
 669
 670         return 0;
 671 }
 672
 673 static int
 674 pagesz_flags(uint64_t page_sz)
 675 {
 676         /* as per mmap() manpage, all page sizes are log2 of page size
 677          * shifted by MAP_HUGE_SHIFT
 678          */
 679         int log2 = rte_log2_u64(page_sz);
 680
 681         return (log2 << HUGE_SHIFT);
 682 }
 683
 684 static void *
 685 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 686 {
 687         void *addr;
 688         int flags;
 689
 690         /* allocate anonymous hugepages */
 691         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 692         if (huge)
 693                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 694
 695         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 696         if (addr == MAP_FAILED)
 697                 return NULL;
 698
 699         return addr;
 700 }
 701
 702 struct extmem_param {
 703         void *addr;
 704         size_t len;
 705         size_t pgsz;
 706         rte_iova_t *iova_table;
 707         unsigned int iova_table_len;
 708 };
 709
 710 static int
 711 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 712                 bool huge)
 713 {
 714         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 715                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 716         unsigned int cur_page, n_pages, pgsz_idx;
 717         size_t mem_sz, cur_pgsz;
 718         rte_iova_t *iovas = NULL;
 719         void *addr;
 720         int ret;
 721
 722         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 723                 /* skip anything that is too big */
 724                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 725                         continue;
 726
 727                 cur_pgsz = pgsizes[pgsz_idx];
 728
 729                 /* if we were told not to allocate hugepages, override */
 730                 if (!huge)
 731                         cur_pgsz = sysconf(_SC_PAGESIZE);
 732
 733                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 734                 if (ret < 0) {
 735                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 736                         return -1;
 737                 }
 738
 739                 /* allocate our memory */
 740                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 741
 742                 /* if we couldn't allocate memory with a specified page size,
 743                  * that doesn't mean we can't do it with other page sizes, so
 744                  * try another one.
 745                  */
 746                 if (addr == NULL)
 747                         continue;
 748
 749                 /* store IOVA addresses for every page in this memory area */
 750                 n_pages = mem_sz / cur_pgsz;
 751
 752                 iovas = malloc(sizeof(*iovas) * n_pages);
 753
 754                 if (iovas == NULL) {
 755                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 756                         goto fail;
 757                 }
 758                 /* lock memory if it's not huge pages */
 759                 if (!huge)
 760                         mlock(addr, mem_sz);
 761
 762                 /* populate IOVA addresses */
 763                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 764                         rte_iova_t iova;
 765                         size_t offset;
 766                         void *cur;
 767
 768                         offset = cur_pgsz * cur_page;
 769                         cur = RTE_PTR_ADD(addr, offset);
 770
 771                         /* touch the page before getting its IOVA */
 772                         *(volatile char *)cur = 0;
 773
 774                         iova = rte_mem_virt2iova(cur);
 775
 776                         iovas[cur_page] = iova;
 777                 }
 778
 779                 break;
 780         }
 781         /* if we couldn't allocate anything */
 782         if (iovas == NULL)
 783                 return -1;
 784
 785         param->addr = addr;
 786         param->len = mem_sz;
 787         param->pgsz = cur_pgsz;
 788         param->iova_table = iovas;
 789         param->iova_table_len = n_pages;
 790
 791         return 0;
 792 fail:
 793         if (iovas)
 794                 free(iovas);
 795         if (addr)
 796                 munmap(addr, mem_sz);
 797
 798         return -1;
 799 }
 800
 801 static int
 802 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 803 {
 804         struct extmem_param param;
 805         int socket_id, ret;
 806
 807         memset(&param, 0, sizeof(param));
 808
 809         /* check if our heap exists */
 810         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 811         if (socket_id < 0) {
 812                 /* create our heap */
 813                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 814                 if (ret < 0) {
 815                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 816                         return -1;
 817                 }
 818         }
 819
 820         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 821         if (ret < 0) {
 822                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 823                 return -1;
 824         }
 825
 826         /* we now have a valid memory area, so add it to heap */
 827         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 828                         param.addr, param.len, param.iova_table,
 829                         param.iova_table_len, param.pgsz);
 830
 831         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 832
 833         /* not needed any more */
 834         free(param.iova_table);
 835
 836         if (ret < 0) {
 837                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 838                 munmap(param.addr, param.len);
 839                 return -1;
 840         }
 841
 842         /* success */
 843
 844         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 845                         param.len >> 20);
 846
 847         return 0;
 848 }
 849 static void
 850 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 851              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 852 {
 853         uint16_t pid = 0;
 854         int ret;
 855
 856         RTE_ETH_FOREACH_DEV(pid) {
 857                 struct rte_eth_dev *dev =
 858                         &rte_eth_devices[pid];
 859
 860                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 861                                         memhdr->len);
 862                 if (ret) {
 863                         TESTPMD_LOG(DEBUG,
 864                                     "unable to DMA unmap addr 0x%p "
 865                                     "for device %s\n",
 866                                     memhdr->addr, dev->data->name);
 867                 }
 868         }
 869         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 870         if (ret) {
 871                 TESTPMD_LOG(DEBUG,
 872                             "unable to un-register addr 0x%p\n", memhdr->addr);
 873         }
 874 }
 875
 876 static void
 877 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 878            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 879 {
 880         uint16_t pid = 0;
 881         size_t page_size = sysconf(_SC_PAGESIZE);
 882         int ret;
 883
 884         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 885                                   page_size);
 886         if (ret) {
 887                 TESTPMD_LOG(DEBUG,
 888                             "unable to register addr 0x%p\n", memhdr->addr);
 889                 return;
 890         }
 891         RTE_ETH_FOREACH_DEV(pid) {
 892                 struct rte_eth_dev *dev =
 893                         &rte_eth_devices[pid];
 894
 895                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 896                                       memhdr->len);
 897                 if (ret) {
 898                         TESTPMD_LOG(DEBUG,
 899                                     "unable to DMA map addr 0x%p "
 900                                     "for device %s\n",
 901                                     memhdr->addr, dev->data->name);
 902                 }
 903         }
 904 }
 905
 906 static unsigned int
 907 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 908             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 909 {
 910         struct rte_pktmbuf_extmem *xmem;
 911         unsigned int ext_num, zone_num, elt_num;
 912         uint16_t elt_size;
 913
 914         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 915         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 916         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 917
 918         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 919         if (xmem == NULL) {
 920                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 921                                  "external buffer descriptors\n");
 922                 *ext_mem = NULL;
 923                 return 0;
 924         }
 925         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 926                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 927                 const struct rte_memzone *mz;
 928                 char mz_name[RTE_MEMZONE_NAMESIZE];
 929                 int ret;
 930
 931                 ret = snprintf(mz_name, sizeof(mz_name),
 932                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 933                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 934                         errno = ENAMETOOLONG;
 935                         ext_num = 0;
 936                         break;
 937                 }
 938                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 939                                                  socket_id,
 940                                                  RTE_MEMZONE_IOVA_CONTIG |
 941                                                  RTE_MEMZONE_1GB |
 942                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 943                                                  EXTBUF_ZONE_SIZE);
 944                 if (mz == NULL) {
 945                         /*
 946                          * The caller exits on external buffer creation
 947                          * error, so there is no need to free memzones.
 948                          */
 949                         errno = ENOMEM;
 950                         ext_num = 0;
 951                         break;
 952                 }
 953                 xseg->buf_ptr = mz->addr;
 954                 xseg->buf_iova = mz->iova;
 955                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 956                 xseg->elt_size = elt_size;
 957         }
 958         if (ext_num == 0 && xmem != NULL) {
 959                 free(xmem);
 960                 xmem = NULL;
 961         }
 962         *ext_mem = xmem;
 963         return ext_num;
 964 }
 965
 966 /*
 967  * Configuration initialisation done once at init time.
 968  */
 969 static struct rte_mempool *
 970 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 971                  unsigned int socket_id, uint16_t size_idx)
 972 {
 973         char pool_name[RTE_MEMPOOL_NAMESIZE];
 974         struct rte_mempool *rte_mp = NULL;
 975         uint32_t mb_size;
 976
 977         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 978         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 979
 980         TESTPMD_LOG(INFO,
 981                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 982                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 983
 984         switch (mp_alloc_type) {
 985         case MP_ALLOC_NATIVE:
 986                 {
 987                         /* wrapper to rte_mempool_create() */
 988                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 989                                         rte_mbuf_best_mempool_ops());
 990                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 991                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 992                         break;
 993                 }
 994         case MP_ALLOC_ANON:
 995                 {
 996                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 997                                 mb_size, (unsigned int) mb_mempool_cache,
 998                                 sizeof(struct rte_pktmbuf_pool_private),
 999                                 socket_id, mempool_flags);
1000                         if (rte_mp == NULL)
1001                                 goto err;
1002
1003                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1004                                 rte_mempool_free(rte_mp);
1005                                 rte_mp = NULL;
1006                                 goto err;
1007                         }
1008                         rte_pktmbuf_pool_init(rte_mp, NULL);
1009                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1010                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1011                         break;
1012                 }
1013         case MP_ALLOC_XMEM:
1014         case MP_ALLOC_XMEM_HUGE:
1015                 {
1016                         int heap_socket;
1017                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1018
1019                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1020                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1021
1022                         heap_socket =
1023                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1024                         if (heap_socket < 0)
1025                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1026
1027                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1028                                         rte_mbuf_best_mempool_ops());
1029                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1030                                         mb_mempool_cache, 0, mbuf_seg_size,
1031                                         heap_socket);
1032                         break;
1033                 }
1034         case MP_ALLOC_XBUF:
1035                 {
1036                         struct rte_pktmbuf_extmem *ext_mem;
1037                         unsigned int ext_num;
1038
1039                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1040                                                socket_id, pool_name, &ext_mem);
1041                         if (ext_num == 0)
1042                                 rte_exit(EXIT_FAILURE,
1043                                          "Can't create pinned data buffers\n");
1044
1045                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1046                                         rte_mbuf_best_mempool_ops());
1047                         rte_mp = rte_pktmbuf_pool_create_extbuf
1048                                         (pool_name, nb_mbuf, mb_mempool_cache,
1049                                          0, mbuf_seg_size, socket_id,
1050                                          ext_mem, ext_num);
1051                         free(ext_mem);
1052                         break;
1053                 }
1054         default:
1055                 {
1056                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1057                 }
1058         }
1059
1060 err:
1061         if (rte_mp == NULL) {
1062                 rte_exit(EXIT_FAILURE,
1063                         "Creation of mbuf pool for socket %u failed: %s\n",
1064                         socket_id, rte_strerror(rte_errno));
1065         } else if (verbose_level > 0) {
1066                 rte_mempool_dump(stdout, rte_mp);
1067         }
1068         return rte_mp;
1069 }
1070
1071 /*
1072  * Check given socket id is valid or not with NUMA mode,
1073  * if valid, return 0, else return -1
1074  */
1075 static int
1076 check_socket_id(const unsigned int socket_id)
1077 {
1078         static int warning_once = 0;
1079
1080         if (new_socket_id(socket_id)) {
1081                 if (!warning_once && numa_support)
1082                         printf("Warning: NUMA should be configured manually by"
1083                                " using --port-numa-config and"
1084                                " --ring-numa-config parameters along with"
1085                                " --numa.\n");
1086                 warning_once = 1;
1087                 return -1;
1088         }
1089         return 0;
1090 }
1091
1092 /*
1093  * Get the allowed maximum number of RX queues.
1094  * *pid return the port id which has minimal value of
1095  * max_rx_queues in all ports.
1096  */
1097 queueid_t
1098 get_allowed_max_nb_rxq(portid_t *pid)
1099 {
1100         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1101         bool max_rxq_valid = false;
1102         portid_t pi;
1103         struct rte_eth_dev_info dev_info;
1104
1105         RTE_ETH_FOREACH_DEV(pi) {
1106                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1107                         continue;
1108
1109                 max_rxq_valid = true;
1110                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1111                         allowed_max_rxq = dev_info.max_rx_queues;
1112                         *pid = pi;
1113                 }
1114         }
1115         return max_rxq_valid ? allowed_max_rxq : 0;
1116 }
1117
1118 /*
1119  * Check input rxq is valid or not.
1120  * If input rxq is not greater than any of maximum number
1121  * of RX queues of all ports, it is valid.
1122  * if valid, return 0, else return -1
1123  */
1124 int
1125 check_nb_rxq(queueid_t rxq)
1126 {
1127         queueid_t allowed_max_rxq;
1128         portid_t pid = 0;
1129
1130         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1131         if (rxq > allowed_max_rxq) {
1132                 printf("Fail: input rxq (%u) can't be greater "
1133                        "than max_rx_queues (%u) of port %u\n",
1134                        rxq,
1135                        allowed_max_rxq,
1136                        pid);
1137                 return -1;
1138         }
1139         return 0;
1140 }
1141
1142 /*
1143  * Get the allowed maximum number of TX queues.
1144  * *pid return the port id which has minimal value of
1145  * max_tx_queues in all ports.
1146  */
1147 queueid_t
1148 get_allowed_max_nb_txq(portid_t *pid)
1149 {
1150         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1151         bool max_txq_valid = false;
1152         portid_t pi;
1153         struct rte_eth_dev_info dev_info;
1154
1155         RTE_ETH_FOREACH_DEV(pi) {
1156                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1157                         continue;
1158
1159                 max_txq_valid = true;
1160                 if (dev_info.max_tx_queues < allowed_max_txq) {
1161                         allowed_max_txq = dev_info.max_tx_queues;
1162                         *pid = pi;
1163                 }
1164         }
1165         return max_txq_valid ? allowed_max_txq : 0;
1166 }
1167
1168 /*
1169  * Check input txq is valid or not.
1170  * If input txq is not greater than any of maximum number
1171  * of TX queues of all ports, it is valid.
1172  * if valid, return 0, else return -1
1173  */
1174 int
1175 check_nb_txq(queueid_t txq)
1176 {
1177         queueid_t allowed_max_txq;
1178         portid_t pid = 0;
1179
1180         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1181         if (txq > allowed_max_txq) {
1182                 printf("Fail: input txq (%u) can't be greater "
1183                        "than max_tx_queues (%u) of port %u\n",
1184                        txq,
1185                        allowed_max_txq,
1186                        pid);
1187                 return -1;
1188         }
1189         return 0;
1190 }
1191
1192 /*
1193  * Get the allowed maximum number of RXDs of every rx queue.
1194  * *pid return the port id which has minimal value of
1195  * max_rxd in all queues of all ports.
1196  */
1197 static uint16_t
1198 get_allowed_max_nb_rxd(portid_t *pid)
1199 {
1200         uint16_t allowed_max_rxd = UINT16_MAX;
1201         portid_t pi;
1202         struct rte_eth_dev_info dev_info;
1203
1204         RTE_ETH_FOREACH_DEV(pi) {
1205                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1206                         continue;
1207
1208                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1209                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1210                         *pid = pi;
1211                 }
1212         }
1213         return allowed_max_rxd;
1214 }
1215
1216 /*
1217  * Get the allowed minimal number of RXDs of every rx queue.
1218  * *pid return the port id which has minimal value of
1219  * min_rxd in all queues of all ports.
1220  */
1221 static uint16_t
1222 get_allowed_min_nb_rxd(portid_t *pid)
1223 {
1224         uint16_t allowed_min_rxd = 0;
1225         portid_t pi;
1226         struct rte_eth_dev_info dev_info;
1227
1228         RTE_ETH_FOREACH_DEV(pi) {
1229                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1230                         continue;
1231
1232                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1233                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1234                         *pid = pi;
1235                 }
1236         }
1237
1238         return allowed_min_rxd;
1239 }
1240
1241 /*
1242  * Check input rxd is valid or not.
1243  * If input rxd is not greater than any of maximum number
1244  * of RXDs of every Rx queues and is not less than any of
1245  * minimal number of RXDs of every Rx queues, it is valid.
1246  * if valid, return 0, else return -1
1247  */
1248 int
1249 check_nb_rxd(queueid_t rxd)
1250 {
1251         uint16_t allowed_max_rxd;
1252         uint16_t allowed_min_rxd;
1253         portid_t pid = 0;
1254
1255         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1256         if (rxd > allowed_max_rxd) {
1257                 printf("Fail: input rxd (%u) can't be greater "
1258                        "than max_rxds (%u) of port %u\n",
1259                        rxd,
1260                        allowed_max_rxd,
1261                        pid);
1262                 return -1;
1263         }
1264
1265         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1266         if (rxd < allowed_min_rxd) {
1267                 printf("Fail: input rxd (%u) can't be less "
1268                        "than min_rxds (%u) of port %u\n",
1269                        rxd,
1270                        allowed_min_rxd,
1271                        pid);
1272                 return -1;
1273         }
1274
1275         return 0;
1276 }
1277
1278 /*
1279  * Get the allowed maximum number of TXDs of every rx queues.
1280  * *pid return the port id which has minimal value of
1281  * max_txd in every tx queue.
1282  */
1283 static uint16_t
1284 get_allowed_max_nb_txd(portid_t *pid)
1285 {
1286         uint16_t allowed_max_txd = UINT16_MAX;
1287         portid_t pi;
1288         struct rte_eth_dev_info dev_info;
1289
1290         RTE_ETH_FOREACH_DEV(pi) {
1291                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1292                         continue;
1293
1294                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1295                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1296                         *pid = pi;
1297                 }
1298         }
1299         return allowed_max_txd;
1300 }
1301
1302 /*
1303  * Get the allowed maximum number of TXDs of every tx queues.
1304  * *pid return the port id which has minimal value of
1305  * min_txd in every tx queue.
1306  */
1307 static uint16_t
1308 get_allowed_min_nb_txd(portid_t *pid)
1309 {
1310         uint16_t allowed_min_txd = 0;
1311         portid_t pi;
1312         struct rte_eth_dev_info dev_info;
1313
1314         RTE_ETH_FOREACH_DEV(pi) {
1315                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1316                         continue;
1317
1318                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1319                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1320                         *pid = pi;
1321                 }
1322         }
1323
1324         return allowed_min_txd;
1325 }
1326
1327 /*
1328  * Check input txd is valid or not.
1329  * If input txd is not greater than any of maximum number
1330  * of TXDs of every Rx queues, it is valid.
1331  * if valid, return 0, else return -1
1332  */
1333 int
1334 check_nb_txd(queueid_t txd)
1335 {
1336         uint16_t allowed_max_txd;
1337         uint16_t allowed_min_txd;
1338         portid_t pid = 0;
1339
1340         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1341         if (txd > allowed_max_txd) {
1342                 printf("Fail: input txd (%u) can't be greater "
1343                        "than max_txds (%u) of port %u\n",
1344                        txd,
1345                        allowed_max_txd,
1346                        pid);
1347                 return -1;
1348         }
1349
1350         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1351         if (txd < allowed_min_txd) {
1352                 printf("Fail: input txd (%u) can't be less "
1353                        "than min_txds (%u) of port %u\n",
1354                        txd,
1355                        allowed_min_txd,
1356                        pid);
1357                 return -1;
1358         }
1359         return 0;
1360 }
1361
1362
1363 /*
1364  * Get the allowed maximum number of hairpin queues.
1365  * *pid return the port id which has minimal value of
1366  * max_hairpin_queues in all ports.
1367  */
1368 queueid_t
1369 get_allowed_max_nb_hairpinq(portid_t *pid)
1370 {
1371         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1372         portid_t pi;
1373         struct rte_eth_hairpin_cap cap;
1374
1375         RTE_ETH_FOREACH_DEV(pi) {
1376                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1377                         *pid = pi;
1378                         return 0;
1379                 }
1380                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1381                         allowed_max_hairpinq = cap.max_nb_queues;
1382                         *pid = pi;
1383                 }
1384         }
1385         return allowed_max_hairpinq;
1386 }
1387
1388 /*
1389  * Check input hairpin is valid or not.
1390  * If input hairpin is not greater than any of maximum number
1391  * of hairpin queues of all ports, it is valid.
1392  * if valid, return 0, else return -1
1393  */
1394 int
1395 check_nb_hairpinq(queueid_t hairpinq)
1396 {
1397         queueid_t allowed_max_hairpinq;
1398         portid_t pid = 0;
1399
1400         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1401         if (hairpinq > allowed_max_hairpinq) {
1402                 printf("Fail: input hairpin (%u) can't be greater "
1403                        "than max_hairpin_queues (%u) of port %u\n",
1404                        hairpinq, allowed_max_hairpinq, pid);
1405                 return -1;
1406         }
1407         return 0;
1408 }
1409
1410 static void
1411 init_config(void)
1412 {
1413         portid_t pid;
1414         struct rte_port *port;
1415         struct rte_mempool *mbp;
1416         unsigned int nb_mbuf_per_pool;
1417         lcoreid_t  lc_id;
1418         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1419         struct rte_gro_param gro_param;
1420         uint32_t gso_types;
1421         uint16_t data_size;
1422         bool warning = 0;
1423         int k;
1424         int ret;
1425
1426         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1427
1428         /* Configuration of logical cores. */
1429         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1430                                 sizeof(struct fwd_lcore *) * nb_lcores,
1431                                 RTE_CACHE_LINE_SIZE);
1432         if (fwd_lcores == NULL) {
1433                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1434                                                         "failed\n", nb_lcores);
1435         }
1436         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1437                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1438                                                sizeof(struct fwd_lcore),
1439                                                RTE_CACHE_LINE_SIZE);
1440                 if (fwd_lcores[lc_id] == NULL) {
1441                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1442                                                                 "failed\n");
1443                 }
1444                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1445         }
1446
1447         RTE_ETH_FOREACH_DEV(pid) {
1448                 port = &ports[pid];
1449                 /* Apply default TxRx configuration for all ports */
1450                 port->dev_conf.txmode = tx_mode;
1451                 port->dev_conf.rxmode = rx_mode;
1452
1453                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1454                 if (ret != 0)
1455                         rte_exit(EXIT_FAILURE,
1456                                  "rte_eth_dev_info_get() failed\n");
1457
1458                 ret = update_jumbo_frame_offload(pid);
1459                 if (ret != 0)
1460                         printf("Updating jumbo frame offload failed for port %u\n",
1461                                 pid);
1462
1463                 if (!(port->dev_info.tx_offload_capa &
1464                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1465                         port->dev_conf.txmode.offloads &=
1466                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1467                 if (numa_support) {
1468                         if (port_numa[pid] != NUMA_NO_CONFIG)
1469                                 port_per_socket[port_numa[pid]]++;
1470                         else {
1471                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1472
1473                                 /*
1474                                  * if socket_id is invalid,
1475                                  * set to the first available socket.
1476                                  */
1477                                 if (check_socket_id(socket_id) < 0)
1478                                         socket_id = socket_ids[0];
1479                                 port_per_socket[socket_id]++;
1480                         }
1481                 }
1482
1483                 /* Apply Rx offloads configuration */
1484                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1485                         port->rx_conf[k].offloads =
1486                                 port->dev_conf.rxmode.offloads;
1487                 /* Apply Tx offloads configuration */
1488                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1489                         port->tx_conf[k].offloads =
1490                                 port->dev_conf.txmode.offloads;
1491
1492                 if (eth_link_speed)
1493                         port->dev_conf.link_speeds = eth_link_speed;
1494
1495                 /* set flag to initialize port/queue */
1496                 port->need_reconfig = 1;
1497                 port->need_reconfig_queues = 1;
1498                 port->tx_metadata = 0;
1499
1500                 /* Check for maximum number of segments per MTU. Accordingly
1501                  * update the mbuf data size.
1502                  */
1503                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1504                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1505                         data_size = rx_mode.max_rx_pkt_len /
1506                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1507
1508                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1509                                                         mbuf_data_size[0]) {
1510                                 mbuf_data_size[0] = data_size +
1511                                                  RTE_PKTMBUF_HEADROOM;
1512                                 warning = 1;
1513                         }
1514                 }
1515         }
1516
1517         if (warning)
1518                 TESTPMD_LOG(WARNING,
1519                             "Configured mbuf size of the first segment %hu\n",
1520                             mbuf_data_size[0]);
1521         /*
1522          * Create pools of mbuf.
1523          * If NUMA support is disabled, create a single pool of mbuf in
1524          * socket 0 memory by default.
1525          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1526          *
1527          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1528          * nb_txd can be configured at run time.
1529          */
1530         if (param_total_num_mbufs)
1531                 nb_mbuf_per_pool = param_total_num_mbufs;
1532         else {
1533                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1534                         (nb_lcores * mb_mempool_cache) +
1535                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1536                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1537         }
1538
1539         if (numa_support) {
1540                 uint8_t i, j;
1541
1542                 for (i = 0; i < num_sockets; i++)
1543                         for (j = 0; j < mbuf_data_size_n; j++)
1544                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1545                                         mbuf_pool_create(mbuf_data_size[j],
1546                                                           nb_mbuf_per_pool,
1547                                                           socket_ids[i], j);
1548         } else {
1549                 uint8_t i;
1550
1551                 for (i = 0; i < mbuf_data_size_n; i++)
1552                         mempools[i] = mbuf_pool_create
1553                                         (mbuf_data_size[i],
1554                                          nb_mbuf_per_pool,
1555                                          socket_num == UMA_NO_CONFIG ?
1556                                          0 : socket_num, i);
1557         }
1558
1559         init_port_config();
1560
1561         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1562                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1563         /*
1564          * Records which Mbuf pool to use by each logical core, if needed.
1565          */
1566         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1567                 mbp = mbuf_pool_find(
1568                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1569
1570                 if (mbp == NULL)
1571                         mbp = mbuf_pool_find(0, 0);
1572                 fwd_lcores[lc_id]->mbp = mbp;
1573                 /* initialize GSO context */
1574                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1575                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1576                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1577                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1578                         RTE_ETHER_CRC_LEN;
1579                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1580         }
1581
1582         /* Configuration of packet forwarding streams. */
1583         if (init_fwd_streams() < 0)
1584                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1585
1586         fwd_config_setup();
1587
1588         /* create a gro context for each lcore */
1589         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1590         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1591         gro_param.max_item_per_flow = MAX_PKT_BURST;
1592         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1593                 gro_param.socket_id = rte_lcore_to_socket_id(
1594                                 fwd_lcores_cpuids[lc_id]);
1595                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1596                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1597                         rte_exit(EXIT_FAILURE,
1598                                         "rte_gro_ctx_create() failed\n");
1599                 }
1600         }
1601 }
1602
1603
1604 void
1605 reconfig(portid_t new_port_id, unsigned socket_id)
1606 {
1607         struct rte_port *port;
1608         int ret;
1609
1610         /* Reconfiguration of Ethernet ports. */
1611         port = &ports[new_port_id];
1612
1613         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1614         if (ret != 0)
1615                 return;
1616
1617         /* set flag to initialize port/queue */
1618         port->need_reconfig = 1;
1619         port->need_reconfig_queues = 1;
1620         port->socket_id = socket_id;
1621
1622         init_port_config();
1623 }
1624
1625
1626 int
1627 init_fwd_streams(void)
1628 {
1629         portid_t pid;
1630         struct rte_port *port;
1631         streamid_t sm_id, nb_fwd_streams_new;
1632         queueid_t q;
1633
1634         /* set socket id according to numa or not */
1635         RTE_ETH_FOREACH_DEV(pid) {
1636                 port = &ports[pid];
1637                 if (nb_rxq > port->dev_info.max_rx_queues) {
1638                         printf("Fail: nb_rxq(%d) is greater than "
1639                                 "max_rx_queues(%d)\n", nb_rxq,
1640                                 port->dev_info.max_rx_queues);
1641                         return -1;
1642                 }
1643                 if (nb_txq > port->dev_info.max_tx_queues) {
1644                         printf("Fail: nb_txq(%d) is greater than "
1645                                 "max_tx_queues(%d)\n", nb_txq,
1646                                 port->dev_info.max_tx_queues);
1647                         return -1;
1648                 }
1649                 if (numa_support) {
1650                         if (port_numa[pid] != NUMA_NO_CONFIG)
1651                                 port->socket_id = port_numa[pid];
1652                         else {
1653                                 port->socket_id = rte_eth_dev_socket_id(pid);
1654
1655                                 /*
1656                                  * if socket_id is invalid,
1657                                  * set to the first available socket.
1658                                  */
1659                                 if (check_socket_id(port->socket_id) < 0)
1660                                         port->socket_id = socket_ids[0];
1661                         }
1662                 }
1663                 else {
1664                         if (socket_num == UMA_NO_CONFIG)
1665                                 port->socket_id = 0;
1666                         else
1667                                 port->socket_id = socket_num;
1668                 }
1669         }
1670
1671         q = RTE_MAX(nb_rxq, nb_txq);
1672         if (q == 0) {
1673                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1674                 return -1;
1675         }
1676         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1677         if (nb_fwd_streams_new == nb_fwd_streams)
1678                 return 0;
1679         /* clear the old */
1680         if (fwd_streams != NULL) {
1681                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1682                         if (fwd_streams[sm_id] == NULL)
1683                                 continue;
1684                         rte_free(fwd_streams[sm_id]);
1685                         fwd_streams[sm_id] = NULL;
1686                 }
1687                 rte_free(fwd_streams);
1688                 fwd_streams = NULL;
1689         }
1690
1691         /* init new */
1692         nb_fwd_streams = nb_fwd_streams_new;
1693         if (nb_fwd_streams) {
1694                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1695                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1696                         RTE_CACHE_LINE_SIZE);
1697                 if (fwd_streams == NULL)
1698                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1699                                  " (struct fwd_stream *)) failed\n",
1700                                  nb_fwd_streams);
1701
1702                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1703                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1704                                 " struct fwd_stream", sizeof(struct fwd_stream),
1705                                 RTE_CACHE_LINE_SIZE);
1706                         if (fwd_streams[sm_id] == NULL)
1707                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1708                                          "(struct fwd_stream) failed\n");
1709                 }
1710         }
1711
1712         return 0;
1713 }
1714
1715 static void
1716 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1717 {
1718         uint64_t total_burst, sburst;
1719         uint64_t nb_burst;
1720         uint64_t burst_stats[4];
1721         uint16_t pktnb_stats[4];
1722         uint16_t nb_pkt;
1723         int burst_percent[4], sburstp;
1724         int i;
1725
1726         /*
1727          * First compute the total number of packet bursts and the
1728          * two highest numbers of bursts of the same number of packets.
1729          */
1730         memset(&burst_stats, 0x0, sizeof(burst_stats));
1731         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1732
1733         /* Show stats for 0 burst size always */
1734         total_burst = pbs->pkt_burst_spread[0];
1735         burst_stats[0] = pbs->pkt_burst_spread[0];
1736         pktnb_stats[0] = 0;
1737
1738         /* Find the next 2 burst sizes with highest occurrences. */
1739         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1740                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1741
1742                 if (nb_burst == 0)
1743                         continue;
1744
1745                 total_burst += nb_burst;
1746
1747                 if (nb_burst > burst_stats[1]) {
1748                         burst_stats[2] = burst_stats[1];
1749                         pktnb_stats[2] = pktnb_stats[1];
1750                         burst_stats[1] = nb_burst;
1751                         pktnb_stats[1] = nb_pkt;
1752                 } else if (nb_burst > burst_stats[2]) {
1753                         burst_stats[2] = nb_burst;
1754                         pktnb_stats[2] = nb_pkt;
1755                 }
1756         }
1757         if (total_burst == 0)
1758                 return;
1759
1760         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1761         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1762                 if (i == 3) {
1763                         printf("%d%% of other]\n", 100 - sburstp);
1764                         return;
1765                 }
1766
1767                 sburst += burst_stats[i];
1768                 if (sburst == total_burst) {
1769                         printf("%d%% of %d pkts]\n",
1770                                 100 - sburstp, (int) pktnb_stats[i]);
1771                         return;
1772                 }
1773
1774                 burst_percent[i] =
1775                         (double)burst_stats[i] / total_burst * 100;
1776                 printf("%d%% of %d pkts + ",
1777                         burst_percent[i], (int) pktnb_stats[i]);
1778                 sburstp += burst_percent[i];
1779         }
1780 }
1781
1782 static void
1783 fwd_stream_stats_display(streamid_t stream_id)
1784 {
1785         struct fwd_stream *fs;
1786         static const char *fwd_top_stats_border = "-------";
1787
1788         fs = fwd_streams[stream_id];
1789         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1790             (fs->fwd_dropped == 0))
1791                 return;
1792         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1793                "TX Port=%2d/Queue=%2d %s\n",
1794                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1795                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1796         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1797                " TX-dropped: %-14"PRIu64,
1798                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1799
1800         /* if checksum mode */
1801         if (cur_fwd_eng == &csum_fwd_engine) {
1802                 printf("  RX- bad IP checksum: %-14"PRIu64
1803                        "  Rx- bad L4 checksum: %-14"PRIu64
1804                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1805                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1806                         fs->rx_bad_outer_l4_csum);
1807                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1808                         fs->rx_bad_outer_ip_csum);
1809         } else {
1810                 printf("\n");
1811         }
1812
1813         if (record_burst_stats) {
1814                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1815                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1816         }
1817 }
1818
1819 void
1820 fwd_stats_display(void)
1821 {
1822         static const char *fwd_stats_border = "----------------------";
1823         static const char *acc_stats_border = "+++++++++++++++";
1824         struct {
1825                 struct fwd_stream *rx_stream;
1826                 struct fwd_stream *tx_stream;
1827                 uint64_t tx_dropped;
1828                 uint64_t rx_bad_ip_csum;
1829                 uint64_t rx_bad_l4_csum;
1830                 uint64_t rx_bad_outer_l4_csum;
1831                 uint64_t rx_bad_outer_ip_csum;
1832         } ports_stats[RTE_MAX_ETHPORTS];
1833         uint64_t total_rx_dropped = 0;
1834         uint64_t total_tx_dropped = 0;
1835         uint64_t total_rx_nombuf = 0;
1836         struct rte_eth_stats stats;
1837         uint64_t fwd_cycles = 0;
1838         uint64_t total_recv = 0;
1839         uint64_t total_xmit = 0;
1840         struct rte_port *port;
1841         streamid_t sm_id;
1842         portid_t pt_id;
1843         int i;
1844
1845         memset(ports_stats, 0, sizeof(ports_stats));
1846
1847         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1848                 struct fwd_stream *fs = fwd_streams[sm_id];
1849
1850                 if (cur_fwd_config.nb_fwd_streams >
1851                     cur_fwd_config.nb_fwd_ports) {
1852                         fwd_stream_stats_display(sm_id);
1853                 } else {
1854                         ports_stats[fs->tx_port].tx_stream = fs;
1855                         ports_stats[fs->rx_port].rx_stream = fs;
1856                 }
1857
1858                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1859
1860                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1861                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1862                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1863                                 fs->rx_bad_outer_l4_csum;
1864                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1865                                 fs->rx_bad_outer_ip_csum;
1866
1867                 if (record_core_cycles)
1868                         fwd_cycles += fs->core_cycles;
1869         }
1870         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1871                 pt_id = fwd_ports_ids[i];
1872                 port = &ports[pt_id];
1873
1874                 rte_eth_stats_get(pt_id, &stats);
1875                 stats.ipackets -= port->stats.ipackets;
1876                 stats.opackets -= port->stats.opackets;
1877                 stats.ibytes -= port->stats.ibytes;
1878                 stats.obytes -= port->stats.obytes;
1879                 stats.imissed -= port->stats.imissed;
1880                 stats.oerrors -= port->stats.oerrors;
1881                 stats.rx_nombuf -= port->stats.rx_nombuf;
1882
1883                 total_recv += stats.ipackets;
1884                 total_xmit += stats.opackets;
1885                 total_rx_dropped += stats.imissed;
1886                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1887                 total_tx_dropped += stats.oerrors;
1888                 total_rx_nombuf  += stats.rx_nombuf;
1889
1890                 printf("\n  %s Forward statistics for port %-2d %s\n",
1891                        fwd_stats_border, pt_id, fwd_stats_border);
1892
1893                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1894                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1895                        stats.ipackets + stats.imissed);
1896
1897                 if (cur_fwd_eng == &csum_fwd_engine) {
1898                         printf("  Bad-ipcsum: %-14"PRIu64
1899                                " Bad-l4csum: %-14"PRIu64
1900                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1901                                ports_stats[pt_id].rx_bad_ip_csum,
1902                                ports_stats[pt_id].rx_bad_l4_csum,
1903                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1904                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1905                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1906                 }
1907                 if (stats.ierrors + stats.rx_nombuf > 0) {
1908                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1909                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1910                 }
1911
1912                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1913                        "TX-total: %-"PRIu64"\n",
1914                        stats.opackets, ports_stats[pt_id].tx_dropped,
1915                        stats.opackets + ports_stats[pt_id].tx_dropped);
1916
1917                 if (record_burst_stats) {
1918                         if (ports_stats[pt_id].rx_stream)
1919                                 pkt_burst_stats_display("RX",
1920                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1921                         if (ports_stats[pt_id].tx_stream)
1922                                 pkt_burst_stats_display("TX",
1923                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1924                 }
1925
1926                 printf("  %s--------------------------------%s\n",
1927                        fwd_stats_border, fwd_stats_border);
1928         }
1929
1930         printf("\n  %s Accumulated forward statistics for all ports"
1931                "%s\n",
1932                acc_stats_border, acc_stats_border);
1933         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1934                "%-"PRIu64"\n"
1935                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1936                "%-"PRIu64"\n",
1937                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1938                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1939         if (total_rx_nombuf > 0)
1940                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1941         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1942                "%s\n",
1943                acc_stats_border, acc_stats_border);
1944         if (record_core_cycles) {
1945 #define CYC_PER_MHZ 1E6
1946                 if (total_recv > 0 || total_xmit > 0) {
1947                         uint64_t total_pkts = 0;
1948                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1949                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1950                                 total_pkts = total_xmit;
1951                         else
1952                                 total_pkts = total_recv;
1953
1954                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1955                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1956                                " MHz Clock\n",
1957                                (double) fwd_cycles / total_pkts,
1958                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1959                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1960                 }
1961         }
1962 }
1963
1964 void
1965 fwd_stats_reset(void)
1966 {
1967         streamid_t sm_id;
1968         portid_t pt_id;
1969         int i;
1970
1971         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1972                 pt_id = fwd_ports_ids[i];
1973                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1974         }
1975         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1976                 struct fwd_stream *fs = fwd_streams[sm_id];
1977
1978                 fs->rx_packets = 0;
1979                 fs->tx_packets = 0;
1980                 fs->fwd_dropped = 0;
1981                 fs->rx_bad_ip_csum = 0;
1982                 fs->rx_bad_l4_csum = 0;
1983                 fs->rx_bad_outer_l4_csum = 0;
1984                 fs->rx_bad_outer_ip_csum = 0;
1985
1986                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1987                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1988                 fs->core_cycles = 0;
1989         }
1990 }
1991
1992 static void
1993 flush_fwd_rx_queues(void)
1994 {
1995         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1996         portid_t  rxp;
1997         portid_t port_id;
1998         queueid_t rxq;
1999         uint16_t  nb_rx;
2000         uint16_t  i;
2001         uint8_t   j;
2002         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2003         uint64_t timer_period;
2004
2005         /* convert to number of cycles */
2006         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2007
2008         for (j = 0; j < 2; j++) {
2009                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2010                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2011                                 port_id = fwd_ports_ids[rxp];
2012                                 /**
2013                                 * testpmd can stuck in the below do while loop
2014                                 * if rte_eth_rx_burst() always returns nonzero
2015                                 * packets. So timer is added to exit this loop
2016                                 * after 1sec timer expiry.
2017                                 */
2018                                 prev_tsc = rte_rdtsc();
2019                                 do {
2020                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2021                                                 pkts_burst, MAX_PKT_BURST);
2022                                         for (i = 0; i < nb_rx; i++)
2023                                                 rte_pktmbuf_free(pkts_burst[i]);
2024
2025                                         cur_tsc = rte_rdtsc();
2026                                         diff_tsc = cur_tsc - prev_tsc;
2027                                         timer_tsc += diff_tsc;
2028                                 } while ((nb_rx > 0) &&
2029                                         (timer_tsc < timer_period));
2030                                 timer_tsc = 0;
2031                         }
2032                 }
2033                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2034         }
2035 }
2036
2037 static void
2038 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2039 {
2040         struct fwd_stream **fsm;
2041         streamid_t nb_fs;
2042         streamid_t sm_id;
2043 #ifdef RTE_LIB_BITRATESTATS
2044         uint64_t tics_per_1sec;
2045         uint64_t tics_datum;
2046         uint64_t tics_current;
2047         uint16_t i, cnt_ports;
2048
2049         cnt_ports = nb_ports;
2050         tics_datum = rte_rdtsc();
2051         tics_per_1sec = rte_get_timer_hz();
2052 #endif
2053         fsm = &fwd_streams[fc->stream_idx];
2054         nb_fs = fc->stream_nb;
2055         do {
2056                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2057                         (*pkt_fwd)(fsm[sm_id]);
2058 #ifdef RTE_LIB_BITRATESTATS
2059                 if (bitrate_enabled != 0 &&
2060                                 bitrate_lcore_id == rte_lcore_id()) {
2061                         tics_current = rte_rdtsc();
2062                         if (tics_current - tics_datum >= tics_per_1sec) {
2063                                 /* Periodic bitrate calculation */
2064                                 for (i = 0; i < cnt_ports; i++)
2065                                         rte_stats_bitrate_calc(bitrate_data,
2066                                                 ports_ids[i]);
2067                                 tics_datum = tics_current;
2068                         }
2069                 }
2070 #endif
2071 #ifdef RTE_LIB_LATENCYSTATS
2072                 if (latencystats_enabled != 0 &&
2073                                 latencystats_lcore_id == rte_lcore_id())
2074                         rte_latencystats_update();
2075 #endif
2076
2077         } while (! fc->stopped);
2078 }
2079
2080 static int
2081 start_pkt_forward_on_core(void *fwd_arg)
2082 {
2083         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2084                              cur_fwd_config.fwd_eng->packet_fwd);
2085         return 0;
2086 }
2087
2088 /*
2089  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2090  * Used to start communication flows in network loopback test configurations.
2091  */
2092 static int
2093 run_one_txonly_burst_on_core(void *fwd_arg)
2094 {
2095         struct fwd_lcore *fwd_lc;
2096         struct fwd_lcore tmp_lcore;
2097
2098         fwd_lc = (struct fwd_lcore *) fwd_arg;
2099         tmp_lcore = *fwd_lc;
2100         tmp_lcore.stopped = 1;
2101         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2102         return 0;
2103 }
2104
2105 /*
2106  * Launch packet forwarding:
2107  *     - Setup per-port forwarding context.
2108  *     - launch logical cores with their forwarding configuration.
2109  */
2110 static void
2111 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2112 {
2113         port_fwd_begin_t port_fwd_begin;
2114         unsigned int i;
2115         unsigned int lc_id;
2116         int diag;
2117
2118         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2119         if (port_fwd_begin != NULL) {
2120                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2121                         (*port_fwd_begin)(fwd_ports_ids[i]);
2122         }
2123         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2124                 lc_id = fwd_lcores_cpuids[i];
2125                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2126                         fwd_lcores[i]->stopped = 0;
2127                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2128                                                      fwd_lcores[i], lc_id);
2129                         if (diag != 0)
2130                                 printf("launch lcore %u failed - diag=%d\n",
2131                                        lc_id, diag);
2132                 }
2133         }
2134 }
2135
2136 /*
2137  * Launch packet forwarding configuration.
2138  */
2139 void
2140 start_packet_forwarding(int with_tx_first)
2141 {
2142         port_fwd_begin_t port_fwd_begin;
2143         port_fwd_end_t  port_fwd_end;
2144         struct rte_port *port;
2145         unsigned int i;
2146         portid_t   pt_id;
2147
2148         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2149                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2150
2151         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2152                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2153
2154         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2155                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2156                 (!nb_rxq || !nb_txq))
2157                 rte_exit(EXIT_FAILURE,
2158                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2159                         cur_fwd_eng->fwd_mode_name);
2160
2161         if (all_ports_started() == 0) {
2162                 printf("Not all ports were started\n");
2163                 return;
2164         }
2165         if (test_done == 0) {
2166                 printf("Packet forwarding already started\n");
2167                 return;
2168         }
2169
2170
2171         if(dcb_test) {
2172                 for (i = 0; i < nb_fwd_ports; i++) {
2173                         pt_id = fwd_ports_ids[i];
2174                         port = &ports[pt_id];
2175                         if (!port->dcb_flag) {
2176                                 printf("In DCB mode, all forwarding ports must "
2177                                        "be configured in this mode.\n");
2178                                 return;
2179                         }
2180                 }
2181                 if (nb_fwd_lcores == 1) {
2182                         printf("In DCB mode,the nb forwarding cores "
2183                                "should be larger than 1.\n");
2184                         return;
2185                 }
2186         }
2187         test_done = 0;
2188
2189         fwd_config_setup();
2190
2191         if(!no_flush_rx)
2192                 flush_fwd_rx_queues();
2193
2194         pkt_fwd_config_display(&cur_fwd_config);
2195         rxtx_config_display();
2196
2197         fwd_stats_reset();
2198         if (with_tx_first) {
2199                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2200                 if (port_fwd_begin != NULL) {
2201                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2202                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2203                 }
2204                 while (with_tx_first--) {
2205                         launch_packet_forwarding(
2206                                         run_one_txonly_burst_on_core);
2207                         rte_eal_mp_wait_lcore();
2208                 }
2209                 port_fwd_end = tx_only_engine.port_fwd_end;
2210                 if (port_fwd_end != NULL) {
2211                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2212                                 (*port_fwd_end)(fwd_ports_ids[i]);
2213                 }
2214         }
2215         launch_packet_forwarding(start_pkt_forward_on_core);
2216 }
2217
2218 void
2219 stop_packet_forwarding(void)
2220 {
2221         port_fwd_end_t port_fwd_end;
2222         lcoreid_t lc_id;
2223         portid_t pt_id;
2224         int i;
2225
2226         if (test_done) {
2227                 printf("Packet forwarding not started\n");
2228                 return;
2229         }
2230         printf("Telling cores to stop...");
2231         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2232                 fwd_lcores[lc_id]->stopped = 1;
2233         printf("\nWaiting for lcores to finish...\n");
2234         rte_eal_mp_wait_lcore();
2235         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2236         if (port_fwd_end != NULL) {
2237                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2238                         pt_id = fwd_ports_ids[i];
2239                         (*port_fwd_end)(pt_id);
2240                 }
2241         }
2242
2243         fwd_stats_display();
2244
2245         printf("\nDone.\n");
2246         test_done = 1;
2247 }
2248
2249 void
2250 dev_set_link_up(portid_t pid)
2251 {
2252         if (rte_eth_dev_set_link_up(pid) < 0)
2253                 printf("\nSet link up fail.\n");
2254 }
2255
2256 void
2257 dev_set_link_down(portid_t pid)
2258 {
2259         if (rte_eth_dev_set_link_down(pid) < 0)
2260                 printf("\nSet link down fail.\n");
2261 }
2262
2263 static int
2264 all_ports_started(void)
2265 {
2266         portid_t pi;
2267         struct rte_port *port;
2268
2269         RTE_ETH_FOREACH_DEV(pi) {
2270                 port = &ports[pi];
2271                 /* Check if there is a port which is not started */
2272                 if ((port->port_status != RTE_PORT_STARTED) &&
2273                         (port->slave_flag == 0))
2274                         return 0;
2275         }
2276
2277         /* No port is not started */
2278         return 1;
2279 }
2280
2281 int
2282 port_is_stopped(portid_t port_id)
2283 {
2284         struct rte_port *port = &ports[port_id];
2285
2286         if ((port->port_status != RTE_PORT_STOPPED) &&
2287             (port->slave_flag == 0))
2288                 return 0;
2289         return 1;
2290 }
2291
2292 int
2293 all_ports_stopped(void)
2294 {
2295         portid_t pi;
2296
2297         RTE_ETH_FOREACH_DEV(pi) {
2298                 if (!port_is_stopped(pi))
2299                         return 0;
2300         }
2301
2302         return 1;
2303 }
2304
2305 int
2306 port_is_started(portid_t port_id)
2307 {
2308         if (port_id_is_invalid(port_id, ENABLED_WARN))
2309                 return 0;
2310
2311         if (ports[port_id].port_status != RTE_PORT_STARTED)
2312                 return 0;
2313
2314         return 1;
2315 }
2316
2317 /* Configure the Rx and Tx hairpin queues for the selected port. */
2318 static int
2319 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2320 {
2321         queueid_t qi;
2322         struct rte_eth_hairpin_conf hairpin_conf = {
2323                 .peer_count = 1,
2324         };
2325         int i;
2326         int diag;
2327         struct rte_port *port = &ports[pi];
2328         uint16_t peer_rx_port = pi;
2329         uint16_t peer_tx_port = pi;
2330         uint32_t manual = 1;
2331         uint32_t tx_exp = hairpin_mode & 0x10;
2332
2333         if (!(hairpin_mode & 0xf)) {
2334                 peer_rx_port = pi;
2335                 peer_tx_port = pi;
2336                 manual = 0;
2337         } else if (hairpin_mode & 0x1) {
2338                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2339                                                        RTE_ETH_DEV_NO_OWNER);
2340                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2341                         peer_tx_port = rte_eth_find_next_owned_by(0,
2342                                                 RTE_ETH_DEV_NO_OWNER);
2343                 if (p_pi != RTE_MAX_ETHPORTS) {
2344                         peer_rx_port = p_pi;
2345                 } else {
2346                         uint16_t next_pi;
2347
2348                         /* Last port will be the peer RX port of the first. */
2349                         RTE_ETH_FOREACH_DEV(next_pi)
2350                                 peer_rx_port = next_pi;
2351                 }
2352                 manual = 1;
2353         } else if (hairpin_mode & 0x2) {
2354                 if (cnt_pi & 0x1) {
2355                         peer_rx_port = p_pi;
2356                 } else {
2357                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2358                                                 RTE_ETH_DEV_NO_OWNER);
2359                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2360                                 peer_rx_port = pi;
2361                 }
2362                 peer_tx_port = peer_rx_port;
2363                 manual = 1;
2364         }
2365
2366         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2367                 hairpin_conf.peers[0].port = peer_rx_port;
2368                 hairpin_conf.peers[0].queue = i + nb_rxq;
2369                 hairpin_conf.manual_bind = !!manual;
2370                 hairpin_conf.tx_explicit = !!tx_exp;
2371                 diag = rte_eth_tx_hairpin_queue_setup
2372                         (pi, qi, nb_txd, &hairpin_conf);
2373                 i++;
2374                 if (diag == 0)
2375                         continue;
2376
2377                 /* Fail to setup rx queue, return */
2378                 if (rte_atomic16_cmpset(&(port->port_status),
2379                                         RTE_PORT_HANDLING,
2380                                         RTE_PORT_STOPPED) == 0)
2381                         printf("Port %d can not be set back "
2382                                         "to stopped\n", pi);
2383                 printf("Fail to configure port %d hairpin "
2384                                 "queues\n", pi);
2385                 /* try to reconfigure queues next time */
2386                 port->need_reconfig_queues = 1;
2387                 return -1;
2388         }
2389         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2390                 hairpin_conf.peers[0].port = peer_tx_port;
2391                 hairpin_conf.peers[0].queue = i + nb_txq;
2392                 hairpin_conf.manual_bind = !!manual;
2393                 hairpin_conf.tx_explicit = !!tx_exp;
2394                 diag = rte_eth_rx_hairpin_queue_setup
2395                         (pi, qi, nb_rxd, &hairpin_conf);
2396                 i++;
2397                 if (diag == 0)
2398                         continue;
2399
2400                 /* Fail to setup rx queue, return */
2401                 if (rte_atomic16_cmpset(&(port->port_status),
2402                                         RTE_PORT_HANDLING,
2403                                         RTE_PORT_STOPPED) == 0)
2404                         printf("Port %d can not be set back "
2405                                         "to stopped\n", pi);
2406                 printf("Fail to configure port %d hairpin "
2407                                 "queues\n", pi);
2408                 /* try to reconfigure queues next time */
2409                 port->need_reconfig_queues = 1;
2410                 return -1;
2411         }
2412         return 0;
2413 }
2414
2415 /* Configure the Rx with optional split. */
2416 int
2417 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2418                uint16_t nb_rx_desc, unsigned int socket_id,
2419                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2420 {
2421         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2422         unsigned int i, mp_n;
2423         int ret;
2424
2425         if (rx_pkt_nb_segs <= 1 ||
2426             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2427                 rx_conf->rx_seg = NULL;
2428                 rx_conf->rx_nseg = 0;
2429                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2430                                              nb_rx_desc, socket_id,
2431                                              rx_conf, mp);
2432                 return ret;
2433         }
2434         for (i = 0; i < rx_pkt_nb_segs; i++) {
2435                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2436                 struct rte_mempool *mpx;
2437                 /*
2438                  * Use last valid pool for the segments with number
2439                  * exceeding the pool index.
2440                  */
2441                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2442                 mpx = mbuf_pool_find(socket_id, mp_n);
2443                 /* Handle zero as mbuf data buffer size. */
2444                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2445                                    rx_pkt_seg_lengths[i] :
2446                                    mbuf_data_size[mp_n];
2447                 rx_seg->offset = i < rx_pkt_nb_offs ?
2448                                    rx_pkt_seg_offsets[i] : 0;
2449                 rx_seg->mp = mpx ? mpx : mp;
2450         }
2451         rx_conf->rx_nseg = rx_pkt_nb_segs;
2452         rx_conf->rx_seg = rx_useg;
2453         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2454                                     socket_id, rx_conf, NULL);
2455         rx_conf->rx_seg = NULL;
2456         rx_conf->rx_nseg = 0;
2457         return ret;
2458 }
2459
2460 int
2461 start_port(portid_t pid)
2462 {
2463         int diag, need_check_link_status = -1;
2464         portid_t pi;
2465         portid_t p_pi = RTE_MAX_ETHPORTS;
2466         portid_t pl[RTE_MAX_ETHPORTS];
2467         portid_t peer_pl[RTE_MAX_ETHPORTS];
2468         uint16_t cnt_pi = 0;
2469         uint16_t cfg_pi = 0;
2470         int peer_pi;
2471         queueid_t qi;
2472         struct rte_port *port;
2473         struct rte_ether_addr mac_addr;
2474         struct rte_eth_hairpin_cap cap;
2475
2476         if (port_id_is_invalid(pid, ENABLED_WARN))
2477                 return 0;
2478
2479         if(dcb_config)
2480                 dcb_test = 1;
2481         RTE_ETH_FOREACH_DEV(pi) {
2482                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2483                         continue;
2484
2485                 need_check_link_status = 0;
2486                 port = &ports[pi];
2487                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2488                                                  RTE_PORT_HANDLING) == 0) {
2489                         printf("Port %d is now not stopped\n", pi);
2490                         continue;
2491                 }
2492
2493                 if (port->need_reconfig > 0) {
2494                         port->need_reconfig = 0;
2495
2496                         if (flow_isolate_all) {
2497                                 int ret = port_flow_isolate(pi, 1);
2498                                 if (ret) {
2499                                         printf("Failed to apply isolated"
2500                                                " mode on port %d\n", pi);
2501                                         return -1;
2502                                 }
2503                         }
2504                         configure_rxtx_dump_callbacks(0);
2505                         printf("Configuring Port %d (socket %u)\n", pi,
2506                                         port->socket_id);
2507                         if (nb_hairpinq > 0 &&
2508                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2509                                 printf("Port %d doesn't support hairpin "
2510                                        "queues\n", pi);
2511                                 return -1;
2512                         }
2513                         /* configure port */
2514                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2515                                                      nb_txq + nb_hairpinq,
2516                                                      &(port->dev_conf));
2517                         if (diag != 0) {
2518                                 if (rte_atomic16_cmpset(&(port->port_status),
2519                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2520                                         printf("Port %d can not be set back "
2521                                                         "to stopped\n", pi);
2522                                 printf("Fail to configure port %d\n", pi);
2523                                 /* try to reconfigure port next time */
2524                                 port->need_reconfig = 1;
2525                                 return -1;
2526                         }
2527                 }
2528                 if (port->need_reconfig_queues > 0) {
2529                         port->need_reconfig_queues = 0;
2530                         /* setup tx queues */
2531                         for (qi = 0; qi < nb_txq; qi++) {
2532                                 if ((numa_support) &&
2533                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2534                                         diag = rte_eth_tx_queue_setup(pi, qi,
2535                                                 port->nb_tx_desc[qi],
2536                                                 txring_numa[pi],
2537                                                 &(port->tx_conf[qi]));
2538                                 else
2539                                         diag = rte_eth_tx_queue_setup(pi, qi,
2540                                                 port->nb_tx_desc[qi],
2541                                                 port->socket_id,
2542                                                 &(port->tx_conf[qi]));
2543
2544                                 if (diag == 0)
2545                                         continue;
2546
2547                                 /* Fail to setup tx queue, return */
2548                                 if (rte_atomic16_cmpset(&(port->port_status),
2549                                                         RTE_PORT_HANDLING,
2550                                                         RTE_PORT_STOPPED) == 0)
2551                                         printf("Port %d can not be set back "
2552                                                         "to stopped\n", pi);
2553                                 printf("Fail to configure port %d tx queues\n",
2554                                        pi);
2555                                 /* try to reconfigure queues next time */
2556                                 port->need_reconfig_queues = 1;
2557                                 return -1;
2558                         }
2559                         for (qi = 0; qi < nb_rxq; qi++) {
2560                                 /* setup rx queues */
2561                                 if ((numa_support) &&
2562                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2563                                         struct rte_mempool * mp =
2564                                                 mbuf_pool_find
2565                                                         (rxring_numa[pi], 0);
2566                                         if (mp == NULL) {
2567                                                 printf("Failed to setup RX queue:"
2568                                                         "No mempool allocation"
2569                                                         " on the socket %d\n",
2570                                                         rxring_numa[pi]);
2571                                                 return -1;
2572                                         }
2573
2574                                         diag = rx_queue_setup(pi, qi,
2575                                              port->nb_rx_desc[qi],
2576                                              rxring_numa[pi],
2577                                              &(port->rx_conf[qi]),
2578                                              mp);
2579                                 } else {
2580                                         struct rte_mempool *mp =
2581                                                 mbuf_pool_find
2582                                                         (port->socket_id, 0);
2583                                         if (mp == NULL) {
2584                                                 printf("Failed to setup RX queue:"
2585                                                         "No mempool allocation"
2586                                                         " on the socket %d\n",
2587                                                         port->socket_id);
2588                                                 return -1;
2589                                         }
2590                                         diag = rx_queue_setup(pi, qi,
2591                                              port->nb_rx_desc[qi],
2592                                              port->socket_id,
2593                                              &(port->rx_conf[qi]),
2594                                              mp);
2595                                 }
2596                                 if (diag == 0)
2597                                         continue;
2598
2599                                 /* Fail to setup rx queue, return */
2600                                 if (rte_atomic16_cmpset(&(port->port_status),
2601                                                         RTE_PORT_HANDLING,
2602                                                         RTE_PORT_STOPPED) == 0)
2603                                         printf("Port %d can not be set back "
2604                                                         "to stopped\n", pi);
2605                                 printf("Fail to configure port %d rx queues\n",
2606                                        pi);
2607                                 /* try to reconfigure queues next time */
2608                                 port->need_reconfig_queues = 1;
2609                                 return -1;
2610                         }
2611                         /* setup hairpin queues */
2612                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2613                                 return -1;
2614                 }
2615                 configure_rxtx_dump_callbacks(verbose_level);
2616                 if (clear_ptypes) {
2617                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2618                                         NULL, 0);
2619                         if (diag < 0)
2620                                 printf(
2621                                 "Port %d: Failed to disable Ptype parsing\n",
2622                                 pi);
2623                 }
2624
2625                 p_pi = pi;
2626                 cnt_pi++;
2627
2628                 /* start port */
2629                 if (rte_eth_dev_start(pi) < 0) {
2630                         printf("Fail to start port %d\n", pi);
2631
2632                         /* Fail to setup rx queue, return */
2633                         if (rte_atomic16_cmpset(&(port->port_status),
2634                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2635                                 printf("Port %d can not be set back to "
2636                                                         "stopped\n", pi);
2637                         continue;
2638                 }
2639
2640                 if (rte_atomic16_cmpset(&(port->port_status),
2641                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2642                         printf("Port %d can not be set into started\n", pi);
2643
2644                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2645                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2646                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2647                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2648                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2649
2650                 /* at least one port started, need checking link status */
2651                 need_check_link_status = 1;
2652
2653                 pl[cfg_pi++] = pi;
2654         }
2655
2656         if (need_check_link_status == 1 && !no_link_check)
2657                 check_all_ports_link_status(RTE_PORT_ALL);
2658         else if (need_check_link_status == 0)
2659                 printf("Please stop the ports first\n");
2660
2661         if (hairpin_mode & 0xf) {
2662                 uint16_t i;
2663                 int j;
2664
2665                 /* bind all started hairpin ports */
2666                 for (i = 0; i < cfg_pi; i++) {
2667                         pi = pl[i];
2668                         /* bind current Tx to all peer Rx */
2669                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2670                                                         RTE_MAX_ETHPORTS, 1);
2671                         if (peer_pi < 0)
2672                                 return peer_pi;
2673                         for (j = 0; j < peer_pi; j++) {
2674                                 if (!port_is_started(peer_pl[j]))
2675                                         continue;
2676                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2677                                 if (diag < 0) {
2678                                         printf("Error during binding hairpin"
2679                                                " Tx port %u to %u: %s\n",
2680                                                pi, peer_pl[j],
2681                                                rte_strerror(-diag));
2682                                         return -1;
2683                                 }
2684                         }
2685                         /* bind all peer Tx to current Rx */
2686                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2687                                                         RTE_MAX_ETHPORTS, 0);
2688                         if (peer_pi < 0)
2689                                 return peer_pi;
2690                         for (j = 0; j < peer_pi; j++) {
2691                                 if (!port_is_started(peer_pl[j]))
2692                                         continue;
2693                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2694                                 if (diag < 0) {
2695                                         printf("Error during binding hairpin"
2696                                                " Tx port %u to %u: %s\n",
2697                                                peer_pl[j], pi,
2698                                                rte_strerror(-diag));
2699                                         return -1;
2700                                 }
2701                         }
2702                 }
2703         }
2704
2705         printf("Done\n");
2706         return 0;
2707 }
2708
2709 void
2710 stop_port(portid_t pid)
2711 {
2712         portid_t pi;
2713         struct rte_port *port;
2714         int need_check_link_status = 0;
2715         portid_t peer_pl[RTE_MAX_ETHPORTS];
2716         int peer_pi;
2717
2718         if (dcb_test) {
2719                 dcb_test = 0;
2720                 dcb_config = 0;
2721         }
2722
2723         if (port_id_is_invalid(pid, ENABLED_WARN))
2724                 return;
2725
2726         printf("Stopping ports...\n");
2727
2728         RTE_ETH_FOREACH_DEV(pi) {
2729                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2730                         continue;
2731
2732                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2733                         printf("Please remove port %d from forwarding configuration.\n", pi);
2734                         continue;
2735                 }
2736
2737                 if (port_is_bonding_slave(pi)) {
2738                         printf("Please remove port %d from bonded device.\n", pi);
2739                         continue;
2740                 }
2741
2742                 port = &ports[pi];
2743                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2744                                                 RTE_PORT_HANDLING) == 0)
2745                         continue;
2746
2747                 if (hairpin_mode & 0xf) {
2748                         int j;
2749
2750                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2751                         /* unbind all peer Tx from current Rx */
2752                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2753                                                         RTE_MAX_ETHPORTS, 0);
2754                         if (peer_pi < 0)
2755                                 continue;
2756                         for (j = 0; j < peer_pi; j++) {
2757                                 if (!port_is_started(peer_pl[j]))
2758                                         continue;
2759                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2760                         }
2761                 }
2762
2763                 if (port->flow_list)
2764                         port_flow_flush(pi);
2765
2766                 if (rte_eth_dev_stop(pi) != 0)
2767                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2768                                 pi);
2769
2770                 if (rte_atomic16_cmpset(&(port->port_status),
2771                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2772                         printf("Port %d can not be set into stopped\n", pi);
2773                 need_check_link_status = 1;
2774         }
2775         if (need_check_link_status && !no_link_check)
2776                 check_all_ports_link_status(RTE_PORT_ALL);
2777
2778         printf("Done\n");
2779 }
2780
2781 static void
2782 remove_invalid_ports_in(portid_t *array, portid_t *total)
2783 {
2784         portid_t i;
2785         portid_t new_total = 0;
2786
2787         for (i = 0; i < *total; i++)
2788                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2789                         array[new_total] = array[i];
2790                         new_total++;
2791                 }
2792         *total = new_total;
2793 }
2794
2795 static void
2796 remove_invalid_ports(void)
2797 {
2798         remove_invalid_ports_in(ports_ids, &nb_ports);
2799         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2800         nb_cfg_ports = nb_fwd_ports;
2801 }
2802
2803 void
2804 close_port(portid_t pid)
2805 {
2806         portid_t pi;
2807         struct rte_port *port;
2808
2809         if (port_id_is_invalid(pid, ENABLED_WARN))
2810                 return;
2811
2812         printf("Closing ports...\n");
2813
2814         RTE_ETH_FOREACH_DEV(pi) {
2815                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2816                         continue;
2817
2818                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2819                         printf("Please remove port %d from forwarding configuration.\n", pi);
2820                         continue;
2821                 }
2822
2823                 if (port_is_bonding_slave(pi)) {
2824                         printf("Please remove port %d from bonded device.\n", pi);
2825                         continue;
2826                 }
2827
2828                 port = &ports[pi];
2829                 if (rte_atomic16_cmpset(&(port->port_status),
2830                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2831                         printf("Port %d is already closed\n", pi);
2832                         continue;
2833                 }
2834
2835                 port_flow_flush(pi);
2836                 rte_eth_dev_close(pi);
2837         }
2838
2839         remove_invalid_ports();
2840         printf("Done\n");
2841 }
2842
2843 void
2844 reset_port(portid_t pid)
2845 {
2846         int diag;
2847         portid_t pi;
2848         struct rte_port *port;
2849
2850         if (port_id_is_invalid(pid, ENABLED_WARN))
2851                 return;
2852
2853         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2854                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2855                 printf("Can not reset port(s), please stop port(s) first.\n");
2856                 return;
2857         }
2858
2859         printf("Resetting ports...\n");
2860
2861         RTE_ETH_FOREACH_DEV(pi) {
2862                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2863                         continue;
2864
2865                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2866                         printf("Please remove port %d from forwarding "
2867                                "configuration.\n", pi);
2868                         continue;
2869                 }
2870
2871                 if (port_is_bonding_slave(pi)) {
2872                         printf("Please remove port %d from bonded device.\n",
2873                                pi);
2874                         continue;
2875                 }
2876
2877                 diag = rte_eth_dev_reset(pi);
2878                 if (diag == 0) {
2879                         port = &ports[pi];
2880                         port->need_reconfig = 1;
2881                         port->need_reconfig_queues = 1;
2882                 } else {
2883                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2884                 }
2885         }
2886
2887         printf("Done\n");
2888 }
2889
2890 void
2891 attach_port(char *identifier)
2892 {
2893         portid_t pi;
2894         struct rte_dev_iterator iterator;
2895
2896         printf("Attaching a new port...\n");
2897
2898         if (identifier == NULL) {
2899                 printf("Invalid parameters are specified\n");
2900                 return;
2901         }
2902
2903         if (rte_dev_probe(identifier) < 0) {
2904                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2905                 return;
2906         }
2907
2908         /* first attach mode: event */
2909         if (setup_on_probe_event) {
2910                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2911                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2912                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2913                                         ports[pi].need_setup != 0)
2914                                 setup_attached_port(pi);
2915                 return;
2916         }
2917
2918         /* second attach mode: iterator */
2919         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2920                 /* setup ports matching the devargs used for probing */
2921                 if (port_is_forwarding(pi))
2922                         continue; /* port was already attached before */
2923                 setup_attached_port(pi);
2924         }
2925 }
2926
2927 static void
2928 setup_attached_port(portid_t pi)
2929 {
2930         unsigned int socket_id;
2931         int ret;
2932
2933         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2934         /* if socket_id is invalid, set to the first available socket. */
2935         if (check_socket_id(socket_id) < 0)
2936                 socket_id = socket_ids[0];
2937         reconfig(pi, socket_id);
2938         ret = rte_eth_promiscuous_enable(pi);
2939         if (ret != 0)
2940                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2941                         pi, rte_strerror(-ret));
2942
2943         ports_ids[nb_ports++] = pi;
2944         fwd_ports_ids[nb_fwd_ports++] = pi;
2945         nb_cfg_ports = nb_fwd_ports;
2946         ports[pi].need_setup = 0;
2947         ports[pi].port_status = RTE_PORT_STOPPED;
2948
2949         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2950         printf("Done\n");
2951 }
2952
2953 static void
2954 detach_device(struct rte_device *dev)
2955 {
2956         portid_t sibling;
2957
2958         if (dev == NULL) {
2959                 printf("Device already removed\n");
2960                 return;
2961         }
2962
2963         printf("Removing a device...\n");
2964
2965         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2966                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2967                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2968                                 printf("Port %u not stopped\n", sibling);
2969                                 return;
2970                         }
2971                         port_flow_flush(sibling);
2972                 }
2973         }
2974
2975         if (rte_dev_remove(dev) < 0) {
2976                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2977                 return;
2978         }
2979         remove_invalid_ports();
2980
2981         printf("Device is detached\n");
2982         printf("Now total ports is %d\n", nb_ports);
2983         printf("Done\n");
2984         return;
2985 }
2986
2987 void
2988 detach_port_device(portid_t port_id)
2989 {
2990         if (port_id_is_invalid(port_id, ENABLED_WARN))
2991                 return;
2992
2993         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2994                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2995                         printf("Port not stopped\n");
2996                         return;
2997                 }
2998                 printf("Port was not closed\n");
2999         }
3000
3001         detach_device(rte_eth_devices[port_id].device);
3002 }
3003
3004 void
3005 detach_devargs(char *identifier)
3006 {
3007         struct rte_dev_iterator iterator;
3008         struct rte_devargs da;
3009         portid_t port_id;
3010
3011         printf("Removing a device...\n");
3012
3013         memset(&da, 0, sizeof(da));
3014         if (rte_devargs_parsef(&da, "%s", identifier)) {
3015                 printf("cannot parse identifier\n");
3016                 if (da.args)
3017                         free(da.args);
3018                 return;
3019         }
3020
3021         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3022                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3023                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3024                                 printf("Port %u not stopped\n", port_id);
3025                                 rte_eth_iterator_cleanup(&iterator);
3026                                 return;
3027                         }
3028                         port_flow_flush(port_id);
3029                 }
3030         }
3031
3032         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3033                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3034                             da.name, da.bus->name);
3035                 return;
3036         }
3037
3038         remove_invalid_ports();
3039
3040         printf("Device %s is detached\n", identifier);
3041         printf("Now total ports is %d\n", nb_ports);
3042         printf("Done\n");
3043 }
3044
3045 void
3046 pmd_test_exit(void)
3047 {
3048         portid_t pt_id;
3049         unsigned int i;
3050         int ret;
3051
3052         if (test_done == 0)
3053                 stop_packet_forwarding();
3054
3055         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3056                 if (mempools[i]) {
3057                         if (mp_alloc_type == MP_ALLOC_ANON)
3058                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3059                                                      NULL);
3060                 }
3061         }
3062         if (ports != NULL) {
3063                 no_link_check = 1;
3064                 RTE_ETH_FOREACH_DEV(pt_id) {
3065                         printf("\nStopping port %d...\n", pt_id);
3066                         fflush(stdout);
3067                         stop_port(pt_id);
3068                 }
3069                 RTE_ETH_FOREACH_DEV(pt_id) {
3070                         printf("\nShutting down port %d...\n", pt_id);
3071                         fflush(stdout);
3072                         close_port(pt_id);
3073                 }
3074         }
3075
3076         if (hot_plug) {
3077                 ret = rte_dev_event_monitor_stop();
3078                 if (ret) {
3079                         RTE_LOG(ERR, EAL,
3080                                 "fail to stop device event monitor.");
3081                         return;
3082                 }
3083
3084                 ret = rte_dev_event_callback_unregister(NULL,
3085                         dev_event_callback, NULL);
3086                 if (ret < 0) {
3087                         RTE_LOG(ERR, EAL,
3088                                 "fail to unregister device event callback.\n");
3089                         return;
3090                 }
3091
3092                 ret = rte_dev_hotplug_handle_disable();
3093                 if (ret) {
3094                         RTE_LOG(ERR, EAL,
3095                                 "fail to disable hotplug handling.\n");
3096                         return;
3097                 }
3098         }
3099         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3100                 if (mempools[i])
3101                         rte_mempool_free(mempools[i]);
3102         }
3103
3104         printf("\nBye...\n");
3105 }
3106
3107 typedef void (*cmd_func_t)(void);
3108 struct pmd_test_command {
3109         const char *cmd_name;
3110         cmd_func_t cmd_func;
3111 };
3112
3113 /* Check the link status of all ports in up to 9s, and print them finally */
3114 static void
3115 check_all_ports_link_status(uint32_t port_mask)
3116 {
3117 #define CHECK_INTERVAL 100 /* 100ms */
3118 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3119         portid_t portid;
3120         uint8_t count, all_ports_up, print_flag = 0;
3121         struct rte_eth_link link;
3122         int ret;
3123         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3124
3125         printf("Checking link statuses...\n");
3126         fflush(stdout);
3127         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3128                 all_ports_up = 1;
3129                 RTE_ETH_FOREACH_DEV(portid) {
3130                         if ((port_mask & (1 << portid)) == 0)
3131                                 continue;
3132                         memset(&link, 0, sizeof(link));
3133                         ret = rte_eth_link_get_nowait(portid, &link);
3134                         if (ret < 0) {
3135                                 all_ports_up = 0;
3136                                 if (print_flag == 1)
3137                                         printf("Port %u link get failed: %s\n",
3138                                                 portid, rte_strerror(-ret));
3139                                 continue;
3140                         }
3141                         /* print link status if flag set */
3142                         if (print_flag == 1) {
3143                                 rte_eth_link_to_str(link_status,
3144                                         sizeof(link_status), &link);
3145                                 printf("Port %d %s\n", portid, link_status);
3146                                 continue;
3147                         }
3148                         /* clear all_ports_up flag if any link down */
3149                         if (link.link_status == ETH_LINK_DOWN) {
3150                                 all_ports_up = 0;
3151                                 break;
3152                         }
3153                 }
3154                 /* after finally printing all link status, get out */
3155                 if (print_flag == 1)
3156                         break;
3157
3158                 if (all_ports_up == 0) {
3159                         fflush(stdout);
3160                         rte_delay_ms(CHECK_INTERVAL);
3161                 }
3162
3163                 /* set the print_flag if all ports up or timeout */
3164                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3165                         print_flag = 1;
3166                 }
3167
3168                 if (lsc_interrupt)
3169                         break;
3170         }
3171 }
3172
3173 static void
3174 rmv_port_callback(void *arg)
3175 {
3176         int need_to_start = 0;
3177         int org_no_link_check = no_link_check;
3178         portid_t port_id = (intptr_t)arg;
3179         struct rte_device *dev;
3180
3181         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3182
3183         if (!test_done && port_is_forwarding(port_id)) {
3184                 need_to_start = 1;
3185                 stop_packet_forwarding();
3186         }
3187         no_link_check = 1;
3188         stop_port(port_id);
3189         no_link_check = org_no_link_check;
3190
3191         /* Save rte_device pointer before closing ethdev port */
3192         dev = rte_eth_devices[port_id].device;
3193         close_port(port_id);
3194         detach_device(dev); /* might be already removed or have more ports */
3195
3196         if (need_to_start)
3197                 start_packet_forwarding(0);
3198 }
3199
3200 /* This function is used by the interrupt thread */
3201 static int
3202 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3203                   void *ret_param)
3204 {
3205         RTE_SET_USED(param);
3206         RTE_SET_USED(ret_param);
3207
3208         if (type >= RTE_ETH_EVENT_MAX) {
3209                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3210                         port_id, __func__, type);
3211                 fflush(stderr);
3212         } else if (event_print_mask & (UINT32_C(1) << type)) {
3213                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3214                         eth_event_desc[type]);
3215                 fflush(stdout);
3216         }
3217
3218         switch (type) {
3219         case RTE_ETH_EVENT_NEW:
3220                 ports[port_id].need_setup = 1;
3221                 ports[port_id].port_status = RTE_PORT_HANDLING;
3222                 break;
3223         case RTE_ETH_EVENT_INTR_RMV:
3224                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3225                         break;
3226                 if (rte_eal_alarm_set(100000,
3227                                 rmv_port_callback, (void *)(intptr_t)port_id))
3228                         fprintf(stderr, "Could not set up deferred device removal\n");
3229                 break;
3230         case RTE_ETH_EVENT_DESTROY:
3231                 ports[port_id].port_status = RTE_PORT_CLOSED;
3232                 printf("Port %u is closed\n", port_id);
3233                 break;
3234         default:
3235                 break;
3236         }
3237         return 0;
3238 }
3239
3240 static int
3241 register_eth_event_callback(void)
3242 {
3243         int ret;
3244         enum rte_eth_event_type event;
3245
3246         for (event = RTE_ETH_EVENT_UNKNOWN;
3247                         event < RTE_ETH_EVENT_MAX; event++) {
3248                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3249                                 event,
3250                                 eth_event_callback,
3251                                 NULL);
3252                 if (ret != 0) {
3253                         TESTPMD_LOG(ERR, "Failed to register callback for "
3254                                         "%s event\n", eth_event_desc[event]);
3255                         return -1;
3256                 }
3257         }
3258
3259         return 0;
3260 }
3261
3262 /* This function is used by the interrupt thread */
3263 static void
3264 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3265                              __rte_unused void *arg)
3266 {
3267         uint16_t port_id;
3268         int ret;
3269
3270         if (type >= RTE_DEV_EVENT_MAX) {
3271                 fprintf(stderr, "%s called upon invalid event %d\n",
3272                         __func__, type);
3273                 fflush(stderr);
3274         }
3275
3276         switch (type) {
3277         case RTE_DEV_EVENT_REMOVE:
3278                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3279                         device_name);
3280                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3281                 if (ret) {
3282                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3283                                 device_name);
3284                         return;
3285                 }
3286                 /*
3287                  * Because the user's callback is invoked in eal interrupt
3288                  * callback, the interrupt callback need to be finished before
3289                  * it can be unregistered when detaching device. So finish
3290                  * callback soon and use a deferred removal to detach device
3291                  * is need. It is a workaround, once the device detaching be
3292                  * moved into the eal in the future, the deferred removal could
3293                  * be deleted.
3294                  */
3295                 if (rte_eal_alarm_set(100000,
3296                                 rmv_port_callback, (void *)(intptr_t)port_id))
3297                         RTE_LOG(ERR, EAL,
3298                                 "Could not set up deferred device removal\n");
3299                 break;
3300         case RTE_DEV_EVENT_ADD:
3301                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3302                         device_name);
3303                 /* TODO: After finish kernel driver binding,
3304                  * begin to attach port.
3305                  */
3306                 break;
3307         default:
3308                 break;
3309         }
3310 }
3311
3312 static void
3313 rxtx_port_config(struct rte_port *port)
3314 {
3315         uint16_t qid;
3316         uint64_t offloads;
3317
3318         for (qid = 0; qid < nb_rxq; qid++) {
3319                 offloads = port->rx_conf[qid].offloads;
3320                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3321                 if (offloads != 0)
3322                         port->rx_conf[qid].offloads = offloads;
3323
3324                 /* Check if any Rx parameters have been passed */
3325                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3326                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3327
3328                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3329                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3330
3331                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3332                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3333
3334                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3335                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3336
3337                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3338                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3339
3340                 port->nb_rx_desc[qid] = nb_rxd;
3341         }
3342
3343         for (qid = 0; qid < nb_txq; qid++) {
3344                 offloads = port->tx_conf[qid].offloads;
3345                 port->tx_conf[qid] = port->dev_info.default_txconf;
3346                 if (offloads != 0)
3347                         port->tx_conf[qid].offloads = offloads;
3348
3349                 /* Check if any Tx parameters have been passed */
3350                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3351                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3352
3353                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3354                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3355
3356                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3357                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3358
3359                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3360                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3361
3362                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3363                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3364
3365                 port->nb_tx_desc[qid] = nb_txd;
3366         }
3367 }
3368
3369 /*
3370  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3371  * MTU is also aligned if JUMBO_FRAME offload is not set.
3372  *
3373  * port->dev_info should be set before calling this function.
3374  *
3375  * return 0 on success, negative on error
3376  */
3377 int
3378 update_jumbo_frame_offload(portid_t portid)
3379 {
3380         struct rte_port *port = &ports[portid];
3381         uint32_t eth_overhead;
3382         uint64_t rx_offloads;
3383         int ret;
3384         bool on;
3385
3386         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3387         if (port->dev_info.max_mtu != UINT16_MAX &&
3388             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3389                 eth_overhead = port->dev_info.max_rx_pktlen -
3390                                 port->dev_info.max_mtu;
3391         else
3392                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3393
3394         rx_offloads = port->dev_conf.rxmode.offloads;
3395
3396         /* Default config value is 0 to use PMD specific overhead */
3397         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3398                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3399
3400         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3401                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3402                 on = false;
3403         } else {
3404                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3405                         printf("Frame size (%u) is not supported by port %u\n",
3406                                 port->dev_conf.rxmode.max_rx_pkt_len,
3407                                 portid);
3408                         return -1;
3409                 }
3410                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3411                 on = true;
3412         }
3413
3414         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3415                 uint16_t qid;
3416
3417                 port->dev_conf.rxmode.offloads = rx_offloads;
3418
3419                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3420                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3421                         if (on)
3422                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3423                         else
3424                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3425                 }
3426         }
3427
3428         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3429          * if unset do it here
3430          */
3431         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3432                 ret = rte_eth_dev_set_mtu(portid,
3433                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3434                 if (ret)
3435                         printf("Failed to set MTU to %u for port %u\n",
3436                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3437                                 portid);
3438         }
3439
3440         return 0;
3441 }
3442
3443 void
3444 init_port_config(void)
3445 {
3446         portid_t pid;
3447         struct rte_port *port;
3448         int ret;
3449
3450         RTE_ETH_FOREACH_DEV(pid) {
3451                 port = &ports[pid];
3452                 port->dev_conf.fdir_conf = fdir_conf;
3453
3454                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3455                 if (ret != 0)
3456                         return;
3457
3458                 if (nb_rxq > 1) {
3459                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3460                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3461                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3462                 } else {
3463                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3464                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3465                 }
3466
3467                 if (port->dcb_flag == 0) {
3468                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3469                                 port->dev_conf.rxmode.mq_mode =
3470                                         (enum rte_eth_rx_mq_mode)
3471                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3472                         else
3473                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3474                 }
3475
3476                 rxtx_port_config(port);
3477
3478                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3479                 if (ret != 0)
3480                         return;
3481
3482 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3483                 rte_pmd_ixgbe_bypass_init(pid);
3484 #endif
3485
3486                 if (lsc_interrupt &&
3487                     (rte_eth_devices[pid].data->dev_flags &
3488                      RTE_ETH_DEV_INTR_LSC))
3489                         port->dev_conf.intr_conf.lsc = 1;
3490                 if (rmv_interrupt &&
3491                     (rte_eth_devices[pid].data->dev_flags &
3492                      RTE_ETH_DEV_INTR_RMV))
3493                         port->dev_conf.intr_conf.rmv = 1;
3494         }
3495 }
3496
3497 void set_port_slave_flag(portid_t slave_pid)
3498 {
3499         struct rte_port *port;
3500
3501         port = &ports[slave_pid];
3502         port->slave_flag = 1;
3503 }
3504
3505 void clear_port_slave_flag(portid_t slave_pid)
3506 {
3507         struct rte_port *port;
3508
3509         port = &ports[slave_pid];
3510         port->slave_flag = 0;
3511 }
3512
3513 uint8_t port_is_bonding_slave(portid_t slave_pid)
3514 {
3515         struct rte_port *port;
3516
3517         port = &ports[slave_pid];
3518         if ((rte_eth_devices[slave_pid].data->dev_flags &
3519             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3520                 return 1;
3521         return 0;
3522 }
3523
3524 const uint16_t vlan_tags[] = {
3525                 0,  1,  2,  3,  4,  5,  6,  7,
3526                 8,  9, 10, 11,  12, 13, 14, 15,
3527                 16, 17, 18, 19, 20, 21, 22, 23,
3528                 24, 25, 26, 27, 28, 29, 30, 31
3529 };
3530
3531 static  int
3532 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3533                  enum dcb_mode_enable dcb_mode,
3534                  enum rte_eth_nb_tcs num_tcs,
3535                  uint8_t pfc_en)
3536 {
3537         uint8_t i;
3538         int32_t rc;
3539         struct rte_eth_rss_conf rss_conf;
3540
3541         /*
3542          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3543          * given above, and the number of traffic classes available for use.
3544          */
3545         if (dcb_mode == DCB_VT_ENABLED) {
3546                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3547                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3548                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3549                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3550
3551                 /* VMDQ+DCB RX and TX configurations */
3552                 vmdq_rx_conf->enable_default_pool = 0;
3553                 vmdq_rx_conf->default_pool = 0;
3554                 vmdq_rx_conf->nb_queue_pools =
3555                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3556                 vmdq_tx_conf->nb_queue_pools =
3557                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3558
3559                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3560                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3561                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3562                         vmdq_rx_conf->pool_map[i].pools =
3563                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3564                 }
3565                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3566                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3567                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3568                 }
3569
3570                 /* set DCB mode of RX and TX of multiple queues */
3571                 eth_conf->rxmode.mq_mode =
3572                                 (enum rte_eth_rx_mq_mode)
3573                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3574                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3575         } else {
3576                 struct rte_eth_dcb_rx_conf *rx_conf =
3577                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3578                 struct rte_eth_dcb_tx_conf *tx_conf =
3579                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3580
3581                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3582
3583                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3584                 if (rc != 0)
3585                         return rc;
3586
3587                 rx_conf->nb_tcs = num_tcs;
3588                 tx_conf->nb_tcs = num_tcs;
3589
3590                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3591                         rx_conf->dcb_tc[i] = i % num_tcs;
3592                         tx_conf->dcb_tc[i] = i % num_tcs;
3593                 }
3594
3595                 eth_conf->rxmode.mq_mode =
3596                                 (enum rte_eth_rx_mq_mode)
3597                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3598                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3599                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3600         }
3601
3602         if (pfc_en)
3603                 eth_conf->dcb_capability_en =
3604                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3605         else
3606                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3607
3608         return 0;
3609 }
3610
3611 int
3612 init_port_dcb_config(portid_t pid,
3613                      enum dcb_mode_enable dcb_mode,
3614                      enum rte_eth_nb_tcs num_tcs,
3615                      uint8_t pfc_en)
3616 {
3617         struct rte_eth_conf port_conf;
3618         struct rte_port *rte_port;
3619         int retval;
3620         uint16_t i;
3621
3622         rte_port = &ports[pid];
3623
3624         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3625         /* Enter DCB configuration status */
3626         dcb_config = 1;
3627
3628         port_conf.rxmode = rte_port->dev_conf.rxmode;
3629         port_conf.txmode = rte_port->dev_conf.txmode;
3630
3631         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3632         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3633         if (retval < 0)
3634                 return retval;
3635         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3636
3637         /* re-configure the device . */
3638         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3639         if (retval < 0)
3640                 return retval;
3641
3642         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3643         if (retval != 0)
3644                 return retval;
3645
3646         /* If dev_info.vmdq_pool_base is greater than 0,
3647          * the queue id of vmdq pools is started after pf queues.
3648          */
3649         if (dcb_mode == DCB_VT_ENABLED &&
3650             rte_port->dev_info.vmdq_pool_base > 0) {
3651                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3652                         " for port %d.", pid);
3653                 return -1;
3654         }
3655
3656         /* Assume the ports in testpmd have the same dcb capability
3657          * and has the same number of rxq and txq in dcb mode
3658          */
3659         if (dcb_mode == DCB_VT_ENABLED) {
3660                 if (rte_port->dev_info.max_vfs > 0) {
3661                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3662                         nb_txq = rte_port->dev_info.nb_tx_queues;
3663                 } else {
3664                         nb_rxq = rte_port->dev_info.max_rx_queues;
3665                         nb_txq = rte_port->dev_info.max_tx_queues;
3666                 }
3667         } else {
3668                 /*if vt is disabled, use all pf queues */
3669                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3670                         nb_rxq = rte_port->dev_info.max_rx_queues;
3671                         nb_txq = rte_port->dev_info.max_tx_queues;
3672                 } else {
3673                         nb_rxq = (queueid_t)num_tcs;
3674                         nb_txq = (queueid_t)num_tcs;
3675
3676                 }
3677         }
3678         rx_free_thresh = 64;
3679
3680         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3681
3682         rxtx_port_config(rte_port);
3683         /* VLAN filter */
3684         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3685         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3686                 rx_vft_set(pid, vlan_tags[i], 1);
3687
3688         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3689         if (retval != 0)
3690                 return retval;
3691
3692         rte_port->dcb_flag = 1;
3693
3694         return 0;
3695 }
3696
3697 static void
3698 init_port(void)
3699 {
3700         int i;
3701
3702         /* Configuration of Ethernet ports. */
3703         ports = rte_zmalloc("testpmd: ports",
3704                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3705                             RTE_CACHE_LINE_SIZE);
3706         if (ports == NULL) {
3707                 rte_exit(EXIT_FAILURE,
3708                                 "rte_zmalloc(%d struct rte_port) failed\n",
3709                                 RTE_MAX_ETHPORTS);
3710         }
3711         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3712                 LIST_INIT(&ports[i].flow_tunnel_list);
3713         /* Initialize ports NUMA structures */
3714         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3715         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3716         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3717 }
3718
3719 static void
3720 force_quit(void)
3721 {
3722         pmd_test_exit();
3723         prompt_exit();
3724 }
3725
3726 static void
3727 print_stats(void)
3728 {
3729         uint8_t i;
3730         const char clr[] = { 27, '[', '2', 'J', '\0' };
3731         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3732
3733         /* Clear screen and move to top left */
3734         printf("%s%s", clr, top_left);
3735
3736         printf("\nPort statistics ====================================");
3737         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3738                 nic_stats_display(fwd_ports_ids[i]);
3739
3740         fflush(stdout);
3741 }
3742
3743 static void
3744 signal_handler(int signum)
3745 {
3746         if (signum == SIGINT || signum == SIGTERM) {
3747                 printf("\nSignal %d received, preparing to exit...\n",
3748                                 signum);
3749 #ifdef RTE_LIB_PDUMP
3750                 /* uninitialize packet capture framework */
3751                 rte_pdump_uninit();
3752 #endif
3753 #ifdef RTE_LIB_LATENCYSTATS
3754                 if (latencystats_enabled != 0)
3755                         rte_latencystats_uninit();
3756 #endif
3757                 force_quit();
3758                 /* Set flag to indicate the force termination. */
3759                 f_quit = 1;
3760                 /* exit with the expected status */
3761                 signal(signum, SIG_DFL);
3762                 kill(getpid(), signum);
3763         }
3764 }
3765
3766 int
3767 main(int argc, char** argv)
3768 {
3769         int diag;
3770         portid_t port_id;
3771         uint16_t count;
3772         int ret;
3773
3774         signal(SIGINT, signal_handler);
3775         signal(SIGTERM, signal_handler);
3776
3777         testpmd_logtype = rte_log_register("testpmd");
3778         if (testpmd_logtype < 0)
3779                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3780         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3781
3782         diag = rte_eal_init(argc, argv);
3783         if (diag < 0)
3784                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3785                          rte_strerror(rte_errno));
3786
3787         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3788                 rte_exit(EXIT_FAILURE,
3789                          "Secondary process type not supported.\n");
3790
3791         ret = register_eth_event_callback();
3792         if (ret != 0)
3793                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3794
3795 #ifdef RTE_LIB_PDUMP
3796         /* initialize packet capture framework */
3797         rte_pdump_init();
3798 #endif
3799
3800         count = 0;
3801         RTE_ETH_FOREACH_DEV(port_id) {
3802                 ports_ids[count] = port_id;
3803                 count++;
3804         }
3805         nb_ports = (portid_t) count;
3806         if (nb_ports == 0)
3807                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3808
3809         /* allocate port structures, and init them */
3810         init_port();
3811
3812         set_def_fwd_config();
3813         if (nb_lcores == 0)
3814                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3815                          "Check the core mask argument\n");
3816
3817         /* Bitrate/latency stats disabled by default */
3818 #ifdef RTE_LIB_BITRATESTATS
3819         bitrate_enabled = 0;
3820 #endif
3821 #ifdef RTE_LIB_LATENCYSTATS
3822         latencystats_enabled = 0;
3823 #endif
3824
3825         /* on FreeBSD, mlockall() is disabled by default */
3826 #ifdef RTE_EXEC_ENV_FREEBSD
3827         do_mlockall = 0;
3828 #else
3829         do_mlockall = 1;
3830 #endif
3831
3832         argc -= diag;
3833         argv += diag;
3834         if (argc > 1)
3835                 launch_args_parse(argc, argv);
3836
3837         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3838                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3839                         strerror(errno));
3840         }
3841
3842         if (tx_first && interactive)
3843                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3844                                 "interactive mode.\n");
3845
3846         if (tx_first && lsc_interrupt) {
3847                 printf("Warning: lsc_interrupt needs to be off when "
3848                                 " using tx_first. Disabling.\n");
3849                 lsc_interrupt = 0;
3850         }
3851
3852         if (!nb_rxq && !nb_txq)
3853                 printf("Warning: Either rx or tx queues should be non-zero\n");
3854
3855         if (nb_rxq > 1 && nb_rxq > nb_txq)
3856                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3857                        "but nb_txq=%d will prevent to fully test it.\n",
3858                        nb_rxq, nb_txq);
3859
3860         init_config();
3861
3862         if (hot_plug) {
3863                 ret = rte_dev_hotplug_handle_enable();
3864                 if (ret) {
3865                         RTE_LOG(ERR, EAL,
3866                                 "fail to enable hotplug handling.");
3867                         return -1;
3868                 }
3869
3870                 ret = rte_dev_event_monitor_start();
3871                 if (ret) {
3872                         RTE_LOG(ERR, EAL,
3873                                 "fail to start device event monitoring.");
3874                         return -1;
3875                 }
3876
3877                 ret = rte_dev_event_callback_register(NULL,
3878                         dev_event_callback, NULL);
3879                 if (ret) {
3880                         RTE_LOG(ERR, EAL,
3881                                 "fail  to register device event callback\n");
3882                         return -1;
3883                 }
3884         }
3885
3886         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3887                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3888
3889         /* set all ports to promiscuous mode by default */
3890         RTE_ETH_FOREACH_DEV(port_id) {
3891                 ret = rte_eth_promiscuous_enable(port_id);
3892                 if (ret != 0)
3893                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3894                                 port_id, rte_strerror(-ret));
3895         }
3896
3897         /* Init metrics library */
3898         rte_metrics_init(rte_socket_id());
3899
3900 #ifdef RTE_LIB_LATENCYSTATS
3901         if (latencystats_enabled != 0) {
3902                 int ret = rte_latencystats_init(1, NULL);
3903                 if (ret)
3904                         printf("Warning: latencystats init()"
3905                                 " returned error %d\n", ret);
3906                 printf("Latencystats running on lcore %d\n",
3907                         latencystats_lcore_id);
3908         }
3909 #endif
3910
3911         /* Setup bitrate stats */
3912 #ifdef RTE_LIB_BITRATESTATS
3913         if (bitrate_enabled != 0) {
3914                 bitrate_data = rte_stats_bitrate_create();
3915                 if (bitrate_data == NULL)
3916                         rte_exit(EXIT_FAILURE,
3917                                 "Could not allocate bitrate data.\n");
3918                 rte_stats_bitrate_reg(bitrate_data);
3919         }
3920 #endif
3921
3922 #ifdef RTE_LIB_CMDLINE
3923         if (strlen(cmdline_filename) != 0)
3924                 cmdline_read_from_file(cmdline_filename);
3925
3926         if (interactive == 1) {
3927                 if (auto_start) {
3928                         printf("Start automatic packet forwarding\n");
3929                         start_packet_forwarding(0);
3930                 }
3931                 prompt();
3932                 pmd_test_exit();
3933         } else
3934 #endif
3935         {
3936                 char c;
3937                 int rc;
3938
3939                 f_quit = 0;
3940
3941                 printf("No commandline core given, start packet forwarding\n");
3942                 start_packet_forwarding(tx_first);
3943                 if (stats_period != 0) {
3944                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3945                         uint64_t timer_period;
3946
3947                         /* Convert to number of cycles */
3948                         timer_period = stats_period * rte_get_timer_hz();
3949
3950                         while (f_quit == 0) {
3951                                 cur_time = rte_get_timer_cycles();
3952                                 diff_time += cur_time - prev_time;
3953
3954                                 if (diff_time >= timer_period) {
3955                                         print_stats();
3956                                         /* Reset the timer */
3957                                         diff_time = 0;
3958                                 }
3959                                 /* Sleep to avoid unnecessary checks */
3960                                 prev_time = cur_time;
3961                                 sleep(1);
3962                         }
3963                 }
3964
3965                 printf("Press enter to exit\n");
3966                 rc = read(0, &c, 1);
3967                 pmd_test_exit();
3968                 if (rc < 0)
3969                         return 1;
3970         }
3971
3972         ret = rte_eal_cleanup();
3973         if (ret != 0)
3974                 rte_exit(EXIT_FAILURE,
3975                          "EAL cleanup failed: %s\n", strerror(-ret));
3976
3977         return EXIT_SUCCESS;
3978 }