app/test-pmd/testpmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <stdarg.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <time.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <errno.h>
  15 #include <stdbool.h>
  16
  17 #include <sys/queue.h>
  18 #include <sys/stat.h>
  19
  20 #include <stdint.h>
  21 #include <unistd.h>
  22 #include <inttypes.h>
  23
  24 #include <rte_common.h>
  25 #include <rte_errno.h>
  26 #include <rte_byteorder.h>
  27 #include <rte_log.h>
  28 #include <rte_debug.h>
  29 #include <rte_cycles.h>
  30 #include <rte_memory.h>
  31 #include <rte_memcpy.h>
  32 #include <rte_launch.h>
  33 #include <rte_eal.h>
  34 #include <rte_alarm.h>
  35 #include <rte_per_lcore.h>
  36 #include <rte_lcore.h>
  37 #include <rte_atomic.h>
  38 #include <rte_branch_prediction.h>
  39 #include <rte_mempool.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_mbuf_pool_ops.h>
  43 #include <rte_interrupts.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_dev.h>
  48 #include <rte_string_fns.h>
  49 #ifdef RTE_NET_IXGBE
  50 #include <rte_pmd_ixgbe.h>
  51 #endif
  52 #ifdef RTE_LIB_PDUMP
  53 #include <rte_pdump.h>
  54 #endif
  55 #include <rte_flow.h>
  56 #include <rte_metrics.h>
  57 #ifdef RTE_LIB_BITRATESTATS
  58 #include <rte_bitrate.h>
  59 #endif
  60 #ifdef RTE_LIB_LATENCYSTATS
  61 #include <rte_latencystats.h>
  62 #endif
  63
  64 #include "testpmd.h"
  65
  66 #ifndef MAP_HUGETLB
  67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  68 #define HUGE_FLAG (0x40000)
  69 #else
  70 #define HUGE_FLAG MAP_HUGETLB
  71 #endif
  72
  73 #ifndef MAP_HUGE_SHIFT
  74 /* older kernels (or FreeBSD) will not have this define */
  75 #define HUGE_SHIFT (26)
  76 #else
  77 #define HUGE_SHIFT MAP_HUGE_SHIFT
  78 #endif
  79
  80 #define EXTMEM_HEAP_NAME "extmem"
  81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
  82
  83 uint16_t verbose_level = 0; /**< Silent by default. */
  84 int testpmd_logtype; /**< Log type for testpmd logs */
  85
  86 /* use main core for command line ? */
  87 uint8_t interactive = 0;
  88 uint8_t auto_start = 0;
  89 uint8_t tx_first;
  90 char cmdline_filename[PATH_MAX] = {0};
  91
  92 /*
  93  * NUMA support configuration.
  94  * When set, the NUMA support attempts to dispatch the allocation of the
  95  * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
  96  * probed ports among the CPU sockets 0 and 1.
  97  * Otherwise, all memory is allocated from CPU socket 0.
  98  */
  99 uint8_t numa_support = 1; /**< numa enabled by default */
 100
 101 /*
 102  * In UMA mode,all memory is allocated from socket 0 if --socket-num is
 103  * not configured.
 104  */
 105 uint8_t socket_num = UMA_NO_CONFIG;
 106
 107 /*
 108  * Select mempool allocation type:
 109  * - native: use regular DPDK memory
 110  * - anon: use regular DPDK memory to create mempool, but populate using
 111  *         anonymous memory (may not be IOVA-contiguous)
 112  * - xmem: use externally allocated hugepage memory
 113  */
 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
 115
 116 /*
 117  * Store specified sockets on which memory pool to be used by ports
 118  * is allocated.
 119  */
 120 uint8_t port_numa[RTE_MAX_ETHPORTS];
 121
 122 /*
 123  * Store specified sockets on which RX ring to be used by ports
 124  * is allocated.
 125  */
 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 127
 128 /*
 129  * Store specified sockets on which TX ring to be used by ports
 130  * is allocated.
 131  */
 132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
 133
 134 /*
 135  * Record the Ethernet address of peer target ports to which packets are
 136  * forwarded.
 137  * Must be instantiated with the ethernet addresses of peer traffic generator
 138  * ports.
 139  */
 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 141 portid_t nb_peer_eth_addrs = 0;
 142
 143 /*
 144  * Probed Target Environment.
 145  */
 146 struct rte_port *ports;        /**< For all probed ethernet ports. */
 147 portid_t nb_ports;             /**< Number of probed ethernet ports. */
 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
 149 lcoreid_t nb_lcores;           /**< Number of probed logical cores. */
 150
 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
 152
 153 /*
 154  * Test Forwarding Configuration.
 155  *    nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
 156  *    nb_fwd_ports  <= nb_cfg_ports  <= nb_ports
 157  */
 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
 160 portid_t  nb_cfg_ports;  /**< Number of configured ports. */
 161 portid_t  nb_fwd_ports;  /**< Number of forwarding ports. */
 162
 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];      /**< Port ids configuration. */
 165
 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
 167 streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 168
 169 /*
 170  * Forwarding engines.
 171  */
 172 struct fwd_engine * fwd_engines[] = {
 173         &io_fwd_engine,
 174         &mac_fwd_engine,
 175         &mac_swap_engine,
 176         &flow_gen_engine,
 177         &rx_only_engine,
 178         &tx_only_engine,
 179         &csum_fwd_engine,
 180         &icmp_echo_engine,
 181         &noisy_vnf_engine,
 182         &five_tuple_swap_fwd_engine,
 183 #ifdef RTE_LIBRTE_IEEE1588
 184         &ieee1588_fwd_engine,
 185 #endif
 186         NULL,
 187 };
 188
 189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
 190 uint16_t mempool_flags;
 191
 192 struct fwd_config cur_fwd_config;
 193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
 194 uint32_t retry_enabled;
 195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
 196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 197
 198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
 199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
 200         DEFAULT_MBUF_DATA_SIZE
 201 }; /**< Mbuf data space size. */
 202 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
 203                                       * specified on command-line. */
 204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
 205
 206 /*
 207  * In container, it cannot terminate the process which running with 'stats-period'
 208  * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
 209  */
 210 uint8_t f_quit;
 211
 212 /*
 213  * Configuration of packet segments used to scatter received packets
 214  * if some of split features is configured.
 215  */
 216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
 217 uint8_t  rx_pkt_nb_segs; /**< Number of segments to split */
 218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
 219 uint8_t  rx_pkt_nb_offs; /**< Number of specified offsets */
 220
 221 /*
 222  * Configuration of packet segments used by the "txonly" processing engine.
 223  */
 224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
 225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
 226         TXONLY_DEF_PACKET_LEN,
 227 };
 228 uint8_t  tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
 229
 230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
 231 /**< Split policy for packets to TX. */
 232
 233 uint8_t txonly_multi_flow;
 234 /**< Whether multiple flows are generated in TXONLY mode. */
 235
 236 uint32_t tx_pkt_times_inter;
 237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
 238
 239 uint32_t tx_pkt_times_intra;
 240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
 241
 242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
 243 uint16_t nb_pkt_flowgen_clones; /**< Number of Tx packet clones to send in flowgen mode. */
 244 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
 245
 246 /* current configuration is in DCB or not,0 means it is not in DCB mode */
 247 uint8_t dcb_config = 0;
 248
 249 /* Whether the dcb is in testing status */
 250 uint8_t dcb_test = 0;
 251
 252 /*
 253  * Configurable number of RX/TX queues.
 254  */
 255 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
 256 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
 257 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
 258
 259 /*
 260  * Configurable number of RX/TX ring descriptors.
 261  * Defaults are supplied by drivers via ethdev.
 262  */
 263 #define RTE_TEST_RX_DESC_DEFAULT 0
 264 #define RTE_TEST_TX_DESC_DEFAULT 0
 265 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
 266 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
 267
 268 #define RTE_PMD_PARAM_UNSET -1
 269 /*
 270  * Configurable values of RX and TX ring threshold registers.
 271  */
 272
 273 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
 274 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
 275 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
 276
 277 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
 278 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
 279 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
 280
 281 /*
 282  * Configurable value of RX free threshold.
 283  */
 284 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
 285
 286 /*
 287  * Configurable value of RX drop enable.
 288  */
 289 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
 290
 291 /*
 292  * Configurable value of TX free threshold.
 293  */
 294 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
 295
 296 /*
 297  * Configurable value of TX RS bit threshold.
 298  */
 299 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
 300
 301 /*
 302  * Configurable value of buffered packets before sending.
 303  */
 304 uint16_t noisy_tx_sw_bufsz;
 305
 306 /*
 307  * Configurable value of packet buffer timeout.
 308  */
 309 uint16_t noisy_tx_sw_buf_flush_time;
 310
 311 /*
 312  * Configurable value for size of VNF internal memory area
 313  * used for simulating noisy neighbour behaviour
 314  */
 315 uint64_t noisy_lkup_mem_sz;
 316
 317 /*
 318  * Configurable value of number of random writes done in
 319  * VNF simulation memory area.
 320  */
 321 uint64_t noisy_lkup_num_writes;
 322
 323 /*
 324  * Configurable value of number of random reads done in
 325  * VNF simulation memory area.
 326  */
 327 uint64_t noisy_lkup_num_reads;
 328
 329 /*
 330  * Configurable value of number of random reads/writes done in
 331  * VNF simulation memory area.
 332  */
 333 uint64_t noisy_lkup_num_reads_writes;
 334
 335 /*
 336  * Receive Side Scaling (RSS) configuration.
 337  */
 338 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
 339
 340 /*
 341  * Port topology configuration
 342  */
 343 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
 344
 345 /*
 346  * Avoids to flush all the RX streams before starts forwarding.
 347  */
 348 uint8_t no_flush_rx = 0; /* flush by default */
 349
 350 /*
 351  * Flow API isolated mode.
 352  */
 353 uint8_t flow_isolate_all;
 354
 355 /*
 356  * Avoids to check link status when starting/stopping a port.
 357  */
 358 uint8_t no_link_check = 0; /* check by default */
 359
 360 /*
 361  * Don't automatically start all ports in interactive mode.
 362  */
 363 uint8_t no_device_start = 0;
 364
 365 /*
 366  * Enable link status change notification
 367  */
 368 uint8_t lsc_interrupt = 1; /* enabled by default */
 369
 370 /*
 371  * Enable device removal notification.
 372  */
 373 uint8_t rmv_interrupt = 1; /* enabled by default */
 374
 375 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
 376
 377 /* After attach, port setup is called on event or by iterator */
 378 bool setup_on_probe_event = true;
 379
 380 /* Clear ptypes on port initialization. */
 381 uint8_t clear_ptypes = true;
 382
 383 /* Hairpin ports configuration mode. */
 384 uint16_t hairpin_mode;
 385
 386 /* Pretty printing of ethdev events */
 387 static const char * const eth_event_desc[] = {
 388         [RTE_ETH_EVENT_UNKNOWN] = "unknown",
 389         [RTE_ETH_EVENT_INTR_LSC] = "link state change",
 390         [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
 391         [RTE_ETH_EVENT_INTR_RESET] = "reset",
 392         [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
 393         [RTE_ETH_EVENT_IPSEC] = "IPsec",
 394         [RTE_ETH_EVENT_MACSEC] = "MACsec",
 395         [RTE_ETH_EVENT_INTR_RMV] = "device removal",
 396         [RTE_ETH_EVENT_NEW] = "device probed",
 397         [RTE_ETH_EVENT_DESTROY] = "device released",
 398         [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
 399         [RTE_ETH_EVENT_MAX] = NULL,
 400 };
 401
 402 /*
 403  * Display or mask ether events
 404  * Default to all events except VF_MBOX
 405  */
 406 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
 407                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
 408                             (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
 409                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
 410                             (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
 411                             (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
 412                             (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
 413                             (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
 414 /*
 415  * Decide if all memory are locked for performance.
 416  */
 417 int do_mlockall = 0;
 418
 419 /*
 420  * NIC bypass mode configuration options.
 421  */
 422
 423 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
 424 /* The NIC bypass watchdog timeout. */
 425 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
 426 #endif
 427
 428
 429 #ifdef RTE_LIB_LATENCYSTATS
 430
 431 /*
 432  * Set when latency stats is enabled in the commandline
 433  */
 434 uint8_t latencystats_enabled;
 435
 436 /*
 437  * Lcore ID to serive latency statistics.
 438  */
 439 lcoreid_t latencystats_lcore_id = -1;
 440
 441 #endif
 442
 443 /*
 444  * Ethernet device configuration.
 445  */
 446 struct rte_eth_rxmode rx_mode = {
 447         /* Default maximum frame length.
 448          * Zero is converted to "RTE_ETHER_MTU + PMD Ethernet overhead"
 449          * in init_config().
 450          */
 451         .max_rx_pkt_len = 0,
 452 };
 453
 454 struct rte_eth_txmode tx_mode = {
 455         .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
 456 };
 457
 458 struct rte_fdir_conf fdir_conf = {
 459         .mode = RTE_FDIR_MODE_NONE,
 460         .pballoc = RTE_FDIR_PBALLOC_64K,
 461         .status = RTE_FDIR_REPORT_STATUS,
 462         .mask = {
 463                 .vlan_tci_mask = 0xFFEF,
 464                 .ipv4_mask     = {
 465                         .src_ip = 0xFFFFFFFF,
 466                         .dst_ip = 0xFFFFFFFF,
 467                 },
 468                 .ipv6_mask     = {
 469                         .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 470                         .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 471                 },
 472                 .src_port_mask = 0xFFFF,
 473                 .dst_port_mask = 0xFFFF,
 474                 .mac_addr_byte_mask = 0xFF,
 475                 .tunnel_type_mask = 1,
 476                 .tunnel_id_mask = 0xFFFFFFFF,
 477         },
 478         .drop_queue = 127,
 479 };
 480
 481 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
 482
 483 /*
 484  * Display zero values by default for xstats
 485  */
 486 uint8_t xstats_hide_zero;
 487
 488 /*
 489  * Measure of CPU cycles disabled by default
 490  */
 491 uint8_t record_core_cycles;
 492
 493 /*
 494  * Display of RX and TX bursts disabled by default
 495  */
 496 uint8_t record_burst_stats;
 497
 498 unsigned int num_sockets = 0;
 499 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
 500
 501 #ifdef RTE_LIB_BITRATESTATS
 502 /* Bitrate statistics */
 503 struct rte_stats_bitrates *bitrate_data;
 504 lcoreid_t bitrate_lcore_id;
 505 uint8_t bitrate_enabled;
 506 #endif
 507
 508 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
 509 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
 510
 511 /*
 512  * hexadecimal bitmask of RX mq mode can be enabled.
 513  */
 514 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
 515
 516 /*
 517  * Used to set forced link speed
 518  */
 519 uint32_t eth_link_speed;
 520
 521 /* Forward function declarations */
 522 static void setup_attached_port(portid_t pi);
 523 static void check_all_ports_link_status(uint32_t port_mask);
 524 static int eth_event_callback(portid_t port_id,
 525                               enum rte_eth_event_type type,
 526                               void *param, void *ret_param);
 527 static void dev_event_callback(const char *device_name,
 528                                 enum rte_dev_event_type type,
 529                                 void *param);
 530
 531 /*
 532  * Check if all the ports are started.
 533  * If yes, return positive value. If not, return zero.
 534  */
 535 static int all_ports_started(void);
 536
 537 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
 538 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
 539
 540 /* Holds the registered mbuf dynamic flags names. */
 541 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
 542
 543 /*
 544  * Helper function to check if socket is already discovered.
 545  * If yes, return positive value. If not, return zero.
 546  */
 547 int
 548 new_socket_id(unsigned int socket_id)
 549 {
 550         unsigned int i;
 551
 552         for (i = 0; i < num_sockets; i++) {
 553                 if (socket_ids[i] == socket_id)
 554                         return 0;
 555         }
 556         return 1;
 557 }
 558
 559 /*
 560  * Setup default configuration.
 561  */
 562 static void
 563 set_default_fwd_lcores_config(void)
 564 {
 565         unsigned int i;
 566         unsigned int nb_lc;
 567         unsigned int sock_num;
 568
 569         nb_lc = 0;
 570         for (i = 0; i < RTE_MAX_LCORE; i++) {
 571                 if (!rte_lcore_is_enabled(i))
 572                         continue;
 573                 sock_num = rte_lcore_to_socket_id(i);
 574                 if (new_socket_id(sock_num)) {
 575                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 576                                 rte_exit(EXIT_FAILURE,
 577                                          "Total sockets greater than %u\n",
 578                                          RTE_MAX_NUMA_NODES);
 579                         }
 580                         socket_ids[num_sockets++] = sock_num;
 581                 }
 582                 if (i == rte_get_main_lcore())
 583                         continue;
 584                 fwd_lcores_cpuids[nb_lc++] = i;
 585         }
 586         nb_lcores = (lcoreid_t) nb_lc;
 587         nb_cfg_lcores = nb_lcores;
 588         nb_fwd_lcores = 1;
 589 }
 590
 591 static void
 592 set_def_peer_eth_addrs(void)
 593 {
 594         portid_t i;
 595
 596         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 597                 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
 598                 peer_eth_addrs[i].addr_bytes[5] = i;
 599         }
 600 }
 601
 602 static void
 603 set_default_fwd_ports_config(void)
 604 {
 605         portid_t pt_id;
 606         int i = 0;
 607
 608         RTE_ETH_FOREACH_DEV(pt_id) {
 609                 fwd_ports_ids[i++] = pt_id;
 610
 611                 /* Update sockets info according to the attached device */
 612                 int socket_id = rte_eth_dev_socket_id(pt_id);
 613                 if (socket_id >= 0 && new_socket_id(socket_id)) {
 614                         if (num_sockets >= RTE_MAX_NUMA_NODES) {
 615                                 rte_exit(EXIT_FAILURE,
 616                                          "Total sockets greater than %u\n",
 617                                          RTE_MAX_NUMA_NODES);
 618                         }
 619                         socket_ids[num_sockets++] = socket_id;
 620                 }
 621         }
 622
 623         nb_cfg_ports = nb_ports;
 624         nb_fwd_ports = nb_ports;
 625 }
 626
 627 void
 628 set_def_fwd_config(void)
 629 {
 630         set_default_fwd_lcores_config();
 631         set_def_peer_eth_addrs();
 632         set_default_fwd_ports_config();
 633 }
 634
 635 /* extremely pessimistic estimation of memory required to create a mempool */
 636 static int
 637 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
 638 {
 639         unsigned int n_pages, mbuf_per_pg, leftover;
 640         uint64_t total_mem, mbuf_mem, obj_sz;
 641
 642         /* there is no good way to predict how much space the mempool will
 643          * occupy because it will allocate chunks on the fly, and some of those
 644          * will come from default DPDK memory while some will come from our
 645          * external memory, so just assume 128MB will be enough for everyone.
 646          */
 647         uint64_t hdr_mem = 128 << 20;
 648
 649         /* account for possible non-contiguousness */
 650         obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
 651         if (obj_sz > pgsz) {
 652                 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
 653                 return -1;
 654         }
 655
 656         mbuf_per_pg = pgsz / obj_sz;
 657         leftover = (nb_mbufs % mbuf_per_pg) > 0;
 658         n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
 659
 660         mbuf_mem = n_pages * pgsz;
 661
 662         total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
 663
 664         if (total_mem > SIZE_MAX) {
 665                 TESTPMD_LOG(ERR, "Memory size too big\n");
 666                 return -1;
 667         }
 668         *out = (size_t)total_mem;
 669
 670         return 0;
 671 }
 672
 673 static int
 674 pagesz_flags(uint64_t page_sz)
 675 {
 676         /* as per mmap() manpage, all page sizes are log2 of page size
 677          * shifted by MAP_HUGE_SHIFT
 678          */
 679         int log2 = rte_log2_u64(page_sz);
 680
 681         return (log2 << HUGE_SHIFT);
 682 }
 683
 684 static void *
 685 alloc_mem(size_t memsz, size_t pgsz, bool huge)
 686 {
 687         void *addr;
 688         int flags;
 689
 690         /* allocate anonymous hugepages */
 691         flags = MAP_ANONYMOUS | MAP_PRIVATE;
 692         if (huge)
 693                 flags |= HUGE_FLAG | pagesz_flags(pgsz);
 694
 695         addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
 696         if (addr == MAP_FAILED)
 697                 return NULL;
 698
 699         return addr;
 700 }
 701
 702 struct extmem_param {
 703         void *addr;
 704         size_t len;
 705         size_t pgsz;
 706         rte_iova_t *iova_table;
 707         unsigned int iova_table_len;
 708 };
 709
 710 static int
 711 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
 712                 bool huge)
 713 {
 714         uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
 715                         RTE_PGSIZE_16M, RTE_PGSIZE_16G};    /* POWER */
 716         unsigned int cur_page, n_pages, pgsz_idx;
 717         size_t mem_sz, cur_pgsz;
 718         rte_iova_t *iovas = NULL;
 719         void *addr;
 720         int ret;
 721
 722         for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
 723                 /* skip anything that is too big */
 724                 if (pgsizes[pgsz_idx] > SIZE_MAX)
 725                         continue;
 726
 727                 cur_pgsz = pgsizes[pgsz_idx];
 728
 729                 /* if we were told not to allocate hugepages, override */
 730                 if (!huge)
 731                         cur_pgsz = sysconf(_SC_PAGESIZE);
 732
 733                 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
 734                 if (ret < 0) {
 735                         TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
 736                         return -1;
 737                 }
 738
 739                 /* allocate our memory */
 740                 addr = alloc_mem(mem_sz, cur_pgsz, huge);
 741
 742                 /* if we couldn't allocate memory with a specified page size,
 743                  * that doesn't mean we can't do it with other page sizes, so
 744                  * try another one.
 745                  */
 746                 if (addr == NULL)
 747                         continue;
 748
 749                 /* store IOVA addresses for every page in this memory area */
 750                 n_pages = mem_sz / cur_pgsz;
 751
 752                 iovas = malloc(sizeof(*iovas) * n_pages);
 753
 754                 if (iovas == NULL) {
 755                         TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
 756                         goto fail;
 757                 }
 758                 /* lock memory if it's not huge pages */
 759                 if (!huge)
 760                         mlock(addr, mem_sz);
 761
 762                 /* populate IOVA addresses */
 763                 for (cur_page = 0; cur_page < n_pages; cur_page++) {
 764                         rte_iova_t iova;
 765                         size_t offset;
 766                         void *cur;
 767
 768                         offset = cur_pgsz * cur_page;
 769                         cur = RTE_PTR_ADD(addr, offset);
 770
 771                         /* touch the page before getting its IOVA */
 772                         *(volatile char *)cur = 0;
 773
 774                         iova = rte_mem_virt2iova(cur);
 775
 776                         iovas[cur_page] = iova;
 777                 }
 778
 779                 break;
 780         }
 781         /* if we couldn't allocate anything */
 782         if (iovas == NULL)
 783                 return -1;
 784
 785         param->addr = addr;
 786         param->len = mem_sz;
 787         param->pgsz = cur_pgsz;
 788         param->iova_table = iovas;
 789         param->iova_table_len = n_pages;
 790
 791         return 0;
 792 fail:
 793         if (iovas)
 794                 free(iovas);
 795         if (addr)
 796                 munmap(addr, mem_sz);
 797
 798         return -1;
 799 }
 800
 801 static int
 802 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
 803 {
 804         struct extmem_param param;
 805         int socket_id, ret;
 806
 807         memset(&param, 0, sizeof(param));
 808
 809         /* check if our heap exists */
 810         socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
 811         if (socket_id < 0) {
 812                 /* create our heap */
 813                 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
 814                 if (ret < 0) {
 815                         TESTPMD_LOG(ERR, "Cannot create heap\n");
 816                         return -1;
 817                 }
 818         }
 819
 820         ret = create_extmem(nb_mbufs, mbuf_sz, &param, huge);
 821         if (ret < 0) {
 822                 TESTPMD_LOG(ERR, "Cannot create memory area\n");
 823                 return -1;
 824         }
 825
 826         /* we now have a valid memory area, so add it to heap */
 827         ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
 828                         param.addr, param.len, param.iova_table,
 829                         param.iova_table_len, param.pgsz);
 830
 831         /* when using VFIO, memory is automatically mapped for DMA by EAL */
 832
 833         /* not needed any more */
 834         free(param.iova_table);
 835
 836         if (ret < 0) {
 837                 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
 838                 munmap(param.addr, param.len);
 839                 return -1;
 840         }
 841
 842         /* success */
 843
 844         TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
 845                         param.len >> 20);
 846
 847         return 0;
 848 }
 849 static void
 850 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 851              struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 852 {
 853         uint16_t pid = 0;
 854         int ret;
 855
 856         RTE_ETH_FOREACH_DEV(pid) {
 857                 struct rte_eth_dev *dev =
 858                         &rte_eth_devices[pid];
 859
 860                 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
 861                                         memhdr->len);
 862                 if (ret) {
 863                         TESTPMD_LOG(DEBUG,
 864                                     "unable to DMA unmap addr 0x%p "
 865                                     "for device %s\n",
 866                                     memhdr->addr, dev->data->name);
 867                 }
 868         }
 869         ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
 870         if (ret) {
 871                 TESTPMD_LOG(DEBUG,
 872                             "unable to un-register addr 0x%p\n", memhdr->addr);
 873         }
 874 }
 875
 876 static void
 877 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
 878            struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
 879 {
 880         uint16_t pid = 0;
 881         size_t page_size = sysconf(_SC_PAGESIZE);
 882         int ret;
 883
 884         ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
 885                                   page_size);
 886         if (ret) {
 887                 TESTPMD_LOG(DEBUG,
 888                             "unable to register addr 0x%p\n", memhdr->addr);
 889                 return;
 890         }
 891         RTE_ETH_FOREACH_DEV(pid) {
 892                 struct rte_eth_dev *dev =
 893                         &rte_eth_devices[pid];
 894
 895                 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
 896                                       memhdr->len);
 897                 if (ret) {
 898                         TESTPMD_LOG(DEBUG,
 899                                     "unable to DMA map addr 0x%p "
 900                                     "for device %s\n",
 901                                     memhdr->addr, dev->data->name);
 902                 }
 903         }
 904 }
 905
 906 static unsigned int
 907 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
 908             char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
 909 {
 910         struct rte_pktmbuf_extmem *xmem;
 911         unsigned int ext_num, zone_num, elt_num;
 912         uint16_t elt_size;
 913
 914         elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
 915         elt_num = EXTBUF_ZONE_SIZE / elt_size;
 916         zone_num = (nb_mbufs + elt_num - 1) / elt_num;
 917
 918         xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
 919         if (xmem == NULL) {
 920                 TESTPMD_LOG(ERR, "Cannot allocate memory for "
 921                                  "external buffer descriptors\n");
 922                 *ext_mem = NULL;
 923                 return 0;
 924         }
 925         for (ext_num = 0; ext_num < zone_num; ext_num++) {
 926                 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
 927                 const struct rte_memzone *mz;
 928                 char mz_name[RTE_MEMZONE_NAMESIZE];
 929                 int ret;
 930
 931                 ret = snprintf(mz_name, sizeof(mz_name),
 932                         RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
 933                 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 934                         errno = ENAMETOOLONG;
 935                         ext_num = 0;
 936                         break;
 937                 }
 938                 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
 939                                                  socket_id,
 940                                                  RTE_MEMZONE_IOVA_CONTIG |
 941                                                  RTE_MEMZONE_1GB |
 942                                                  RTE_MEMZONE_SIZE_HINT_ONLY,
 943                                                  EXTBUF_ZONE_SIZE);
 944                 if (mz == NULL) {
 945                         /*
 946                          * The caller exits on external buffer creation
 947                          * error, so there is no need to free memzones.
 948                          */
 949                         errno = ENOMEM;
 950                         ext_num = 0;
 951                         break;
 952                 }
 953                 xseg->buf_ptr = mz->addr;
 954                 xseg->buf_iova = mz->iova;
 955                 xseg->buf_len = EXTBUF_ZONE_SIZE;
 956                 xseg->elt_size = elt_size;
 957         }
 958         if (ext_num == 0 && xmem != NULL) {
 959                 free(xmem);
 960                 xmem = NULL;
 961         }
 962         *ext_mem = xmem;
 963         return ext_num;
 964 }
 965
 966 /*
 967  * Configuration initialisation done once at init time.
 968  */
 969 static struct rte_mempool *
 970 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 971                  unsigned int socket_id, uint16_t size_idx)
 972 {
 973         char pool_name[RTE_MEMPOOL_NAMESIZE];
 974         struct rte_mempool *rte_mp = NULL;
 975         uint32_t mb_size;
 976
 977         mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 978         mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 979
 980         TESTPMD_LOG(INFO,
 981                 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 982                 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
 983
 984         switch (mp_alloc_type) {
 985         case MP_ALLOC_NATIVE:
 986                 {
 987                         /* wrapper to rte_mempool_create() */
 988                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
 989                                         rte_mbuf_best_mempool_ops());
 990                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 991                                 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
 992                         break;
 993                 }
 994         case MP_ALLOC_ANON:
 995                 {
 996                         rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
 997                                 mb_size, (unsigned int) mb_mempool_cache,
 998                                 sizeof(struct rte_pktmbuf_pool_private),
 999                                 socket_id, mempool_flags);
1000                         if (rte_mp == NULL)
1001                                 goto err;
1002
1003                         if (rte_mempool_populate_anon(rte_mp) == 0) {
1004                                 rte_mempool_free(rte_mp);
1005                                 rte_mp = NULL;
1006                                 goto err;
1007                         }
1008                         rte_pktmbuf_pool_init(rte_mp, NULL);
1009                         rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1010                         rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1011                         break;
1012                 }
1013         case MP_ALLOC_XMEM:
1014         case MP_ALLOC_XMEM_HUGE:
1015                 {
1016                         int heap_socket;
1017                         bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1018
1019                         if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1020                                 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1021
1022                         heap_socket =
1023                                 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1024                         if (heap_socket < 0)
1025                                 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1026
1027                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1028                                         rte_mbuf_best_mempool_ops());
1029                         rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1030                                         mb_mempool_cache, 0, mbuf_seg_size,
1031                                         heap_socket);
1032                         break;
1033                 }
1034         case MP_ALLOC_XBUF:
1035                 {
1036                         struct rte_pktmbuf_extmem *ext_mem;
1037                         unsigned int ext_num;
1038
1039                         ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1040                                                socket_id, pool_name, &ext_mem);
1041                         if (ext_num == 0)
1042                                 rte_exit(EXIT_FAILURE,
1043                                          "Can't create pinned data buffers\n");
1044
1045                         TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1046                                         rte_mbuf_best_mempool_ops());
1047                         rte_mp = rte_pktmbuf_pool_create_extbuf
1048                                         (pool_name, nb_mbuf, mb_mempool_cache,
1049                                          0, mbuf_seg_size, socket_id,
1050                                          ext_mem, ext_num);
1051                         free(ext_mem);
1052                         break;
1053                 }
1054         default:
1055                 {
1056                         rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1057                 }
1058         }
1059
1060 err:
1061         if (rte_mp == NULL) {
1062                 rte_exit(EXIT_FAILURE,
1063                         "Creation of mbuf pool for socket %u failed: %s\n",
1064                         socket_id, rte_strerror(rte_errno));
1065         } else if (verbose_level > 0) {
1066                 rte_mempool_dump(stdout, rte_mp);
1067         }
1068         return rte_mp;
1069 }
1070
1071 /*
1072  * Check given socket id is valid or not with NUMA mode,
1073  * if valid, return 0, else return -1
1074  */
1075 static int
1076 check_socket_id(const unsigned int socket_id)
1077 {
1078         static int warning_once = 0;
1079
1080         if (new_socket_id(socket_id)) {
1081                 if (!warning_once && numa_support)
1082                         printf("Warning: NUMA should be configured manually by"
1083                                " using --port-numa-config and"
1084                                " --ring-numa-config parameters along with"
1085                                " --numa.\n");
1086                 warning_once = 1;
1087                 return -1;
1088         }
1089         return 0;
1090 }
1091
1092 /*
1093  * Get the allowed maximum number of RX queues.
1094  * *pid return the port id which has minimal value of
1095  * max_rx_queues in all ports.
1096  */
1097 queueid_t
1098 get_allowed_max_nb_rxq(portid_t *pid)
1099 {
1100         queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1101         bool max_rxq_valid = false;
1102         portid_t pi;
1103         struct rte_eth_dev_info dev_info;
1104
1105         RTE_ETH_FOREACH_DEV(pi) {
1106                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1107                         continue;
1108
1109                 max_rxq_valid = true;
1110                 if (dev_info.max_rx_queues < allowed_max_rxq) {
1111                         allowed_max_rxq = dev_info.max_rx_queues;
1112                         *pid = pi;
1113                 }
1114         }
1115         return max_rxq_valid ? allowed_max_rxq : 0;
1116 }
1117
1118 /*
1119  * Check input rxq is valid or not.
1120  * If input rxq is not greater than any of maximum number
1121  * of RX queues of all ports, it is valid.
1122  * if valid, return 0, else return -1
1123  */
1124 int
1125 check_nb_rxq(queueid_t rxq)
1126 {
1127         queueid_t allowed_max_rxq;
1128         portid_t pid = 0;
1129
1130         allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1131         if (rxq > allowed_max_rxq) {
1132                 printf("Fail: input rxq (%u) can't be greater "
1133                        "than max_rx_queues (%u) of port %u\n",
1134                        rxq,
1135                        allowed_max_rxq,
1136                        pid);
1137                 return -1;
1138         }
1139         return 0;
1140 }
1141
1142 /*
1143  * Get the allowed maximum number of TX queues.
1144  * *pid return the port id which has minimal value of
1145  * max_tx_queues in all ports.
1146  */
1147 queueid_t
1148 get_allowed_max_nb_txq(portid_t *pid)
1149 {
1150         queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1151         bool max_txq_valid = false;
1152         portid_t pi;
1153         struct rte_eth_dev_info dev_info;
1154
1155         RTE_ETH_FOREACH_DEV(pi) {
1156                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1157                         continue;
1158
1159                 max_txq_valid = true;
1160                 if (dev_info.max_tx_queues < allowed_max_txq) {
1161                         allowed_max_txq = dev_info.max_tx_queues;
1162                         *pid = pi;
1163                 }
1164         }
1165         return max_txq_valid ? allowed_max_txq : 0;
1166 }
1167
1168 /*
1169  * Check input txq is valid or not.
1170  * If input txq is not greater than any of maximum number
1171  * of TX queues of all ports, it is valid.
1172  * if valid, return 0, else return -1
1173  */
1174 int
1175 check_nb_txq(queueid_t txq)
1176 {
1177         queueid_t allowed_max_txq;
1178         portid_t pid = 0;
1179
1180         allowed_max_txq = get_allowed_max_nb_txq(&pid);
1181         if (txq > allowed_max_txq) {
1182                 printf("Fail: input txq (%u) can't be greater "
1183                        "than max_tx_queues (%u) of port %u\n",
1184                        txq,
1185                        allowed_max_txq,
1186                        pid);
1187                 return -1;
1188         }
1189         return 0;
1190 }
1191
1192 /*
1193  * Get the allowed maximum number of RXDs of every rx queue.
1194  * *pid return the port id which has minimal value of
1195  * max_rxd in all queues of all ports.
1196  */
1197 static uint16_t
1198 get_allowed_max_nb_rxd(portid_t *pid)
1199 {
1200         uint16_t allowed_max_rxd = UINT16_MAX;
1201         portid_t pi;
1202         struct rte_eth_dev_info dev_info;
1203
1204         RTE_ETH_FOREACH_DEV(pi) {
1205                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1206                         continue;
1207
1208                 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1209                         allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1210                         *pid = pi;
1211                 }
1212         }
1213         return allowed_max_rxd;
1214 }
1215
1216 /*
1217  * Get the allowed minimal number of RXDs of every rx queue.
1218  * *pid return the port id which has minimal value of
1219  * min_rxd in all queues of all ports.
1220  */
1221 static uint16_t
1222 get_allowed_min_nb_rxd(portid_t *pid)
1223 {
1224         uint16_t allowed_min_rxd = 0;
1225         portid_t pi;
1226         struct rte_eth_dev_info dev_info;
1227
1228         RTE_ETH_FOREACH_DEV(pi) {
1229                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1230                         continue;
1231
1232                 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1233                         allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1234                         *pid = pi;
1235                 }
1236         }
1237
1238         return allowed_min_rxd;
1239 }
1240
1241 /*
1242  * Check input rxd is valid or not.
1243  * If input rxd is not greater than any of maximum number
1244  * of RXDs of every Rx queues and is not less than any of
1245  * minimal number of RXDs of every Rx queues, it is valid.
1246  * if valid, return 0, else return -1
1247  */
1248 int
1249 check_nb_rxd(queueid_t rxd)
1250 {
1251         uint16_t allowed_max_rxd;
1252         uint16_t allowed_min_rxd;
1253         portid_t pid = 0;
1254
1255         allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1256         if (rxd > allowed_max_rxd) {
1257                 printf("Fail: input rxd (%u) can't be greater "
1258                        "than max_rxds (%u) of port %u\n",
1259                        rxd,
1260                        allowed_max_rxd,
1261                        pid);
1262                 return -1;
1263         }
1264
1265         allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1266         if (rxd < allowed_min_rxd) {
1267                 printf("Fail: input rxd (%u) can't be less "
1268                        "than min_rxds (%u) of port %u\n",
1269                        rxd,
1270                        allowed_min_rxd,
1271                        pid);
1272                 return -1;
1273         }
1274
1275         return 0;
1276 }
1277
1278 /*
1279  * Get the allowed maximum number of TXDs of every rx queues.
1280  * *pid return the port id which has minimal value of
1281  * max_txd in every tx queue.
1282  */
1283 static uint16_t
1284 get_allowed_max_nb_txd(portid_t *pid)
1285 {
1286         uint16_t allowed_max_txd = UINT16_MAX;
1287         portid_t pi;
1288         struct rte_eth_dev_info dev_info;
1289
1290         RTE_ETH_FOREACH_DEV(pi) {
1291                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1292                         continue;
1293
1294                 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1295                         allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1296                         *pid = pi;
1297                 }
1298         }
1299         return allowed_max_txd;
1300 }
1301
1302 /*
1303  * Get the allowed maximum number of TXDs of every tx queues.
1304  * *pid return the port id which has minimal value of
1305  * min_txd in every tx queue.
1306  */
1307 static uint16_t
1308 get_allowed_min_nb_txd(portid_t *pid)
1309 {
1310         uint16_t allowed_min_txd = 0;
1311         portid_t pi;
1312         struct rte_eth_dev_info dev_info;
1313
1314         RTE_ETH_FOREACH_DEV(pi) {
1315                 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1316                         continue;
1317
1318                 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1319                         allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1320                         *pid = pi;
1321                 }
1322         }
1323
1324         return allowed_min_txd;
1325 }
1326
1327 /*
1328  * Check input txd is valid or not.
1329  * If input txd is not greater than any of maximum number
1330  * of TXDs of every Rx queues, it is valid.
1331  * if valid, return 0, else return -1
1332  */
1333 int
1334 check_nb_txd(queueid_t txd)
1335 {
1336         uint16_t allowed_max_txd;
1337         uint16_t allowed_min_txd;
1338         portid_t pid = 0;
1339
1340         allowed_max_txd = get_allowed_max_nb_txd(&pid);
1341         if (txd > allowed_max_txd) {
1342                 printf("Fail: input txd (%u) can't be greater "
1343                        "than max_txds (%u) of port %u\n",
1344                        txd,
1345                        allowed_max_txd,
1346                        pid);
1347                 return -1;
1348         }
1349
1350         allowed_min_txd = get_allowed_min_nb_txd(&pid);
1351         if (txd < allowed_min_txd) {
1352                 printf("Fail: input txd (%u) can't be less "
1353                        "than min_txds (%u) of port %u\n",
1354                        txd,
1355                        allowed_min_txd,
1356                        pid);
1357                 return -1;
1358         }
1359         return 0;
1360 }
1361
1362
1363 /*
1364  * Get the allowed maximum number of hairpin queues.
1365  * *pid return the port id which has minimal value of
1366  * max_hairpin_queues in all ports.
1367  */
1368 queueid_t
1369 get_allowed_max_nb_hairpinq(portid_t *pid)
1370 {
1371         queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1372         portid_t pi;
1373         struct rte_eth_hairpin_cap cap;
1374
1375         RTE_ETH_FOREACH_DEV(pi) {
1376                 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1377                         *pid = pi;
1378                         return 0;
1379                 }
1380                 if (cap.max_nb_queues < allowed_max_hairpinq) {
1381                         allowed_max_hairpinq = cap.max_nb_queues;
1382                         *pid = pi;
1383                 }
1384         }
1385         return allowed_max_hairpinq;
1386 }
1387
1388 /*
1389  * Check input hairpin is valid or not.
1390  * If input hairpin is not greater than any of maximum number
1391  * of hairpin queues of all ports, it is valid.
1392  * if valid, return 0, else return -1
1393  */
1394 int
1395 check_nb_hairpinq(queueid_t hairpinq)
1396 {
1397         queueid_t allowed_max_hairpinq;
1398         portid_t pid = 0;
1399
1400         allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1401         if (hairpinq > allowed_max_hairpinq) {
1402                 printf("Fail: input hairpin (%u) can't be greater "
1403                        "than max_hairpin_queues (%u) of port %u\n",
1404                        hairpinq, allowed_max_hairpinq, pid);
1405                 return -1;
1406         }
1407         return 0;
1408 }
1409
1410 static void
1411 init_config(void)
1412 {
1413         portid_t pid;
1414         struct rte_port *port;
1415         struct rte_mempool *mbp;
1416         unsigned int nb_mbuf_per_pool;
1417         lcoreid_t  lc_id;
1418         uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1419         struct rte_gro_param gro_param;
1420         uint32_t gso_types;
1421         uint16_t data_size;
1422         bool warning = 0;
1423         int k;
1424         int ret;
1425
1426         memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1427
1428         /* Configuration of logical cores. */
1429         fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1430                                 sizeof(struct fwd_lcore *) * nb_lcores,
1431                                 RTE_CACHE_LINE_SIZE);
1432         if (fwd_lcores == NULL) {
1433                 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1434                                                         "failed\n", nb_lcores);
1435         }
1436         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1437                 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1438                                                sizeof(struct fwd_lcore),
1439                                                RTE_CACHE_LINE_SIZE);
1440                 if (fwd_lcores[lc_id] == NULL) {
1441                         rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1442                                                                 "failed\n");
1443                 }
1444                 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1445         }
1446
1447         RTE_ETH_FOREACH_DEV(pid) {
1448                 port = &ports[pid];
1449                 /* Apply default TxRx configuration for all ports */
1450                 port->dev_conf.txmode = tx_mode;
1451                 port->dev_conf.rxmode = rx_mode;
1452
1453                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1454                 if (ret != 0)
1455                         rte_exit(EXIT_FAILURE,
1456                                  "rte_eth_dev_info_get() failed\n");
1457
1458                 ret = update_jumbo_frame_offload(pid);
1459                 if (ret != 0)
1460                         printf("Updating jumbo frame offload failed for port %u\n",
1461                                 pid);
1462
1463                 if (!(port->dev_info.tx_offload_capa &
1464                       DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1465                         port->dev_conf.txmode.offloads &=
1466                                 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1467                 if (numa_support) {
1468                         if (port_numa[pid] != NUMA_NO_CONFIG)
1469                                 port_per_socket[port_numa[pid]]++;
1470                         else {
1471                                 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1472
1473                                 /*
1474                                  * if socket_id is invalid,
1475                                  * set to the first available socket.
1476                                  */
1477                                 if (check_socket_id(socket_id) < 0)
1478                                         socket_id = socket_ids[0];
1479                                 port_per_socket[socket_id]++;
1480                         }
1481                 }
1482
1483                 /* Apply Rx offloads configuration */
1484                 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1485                         port->rx_conf[k].offloads =
1486                                 port->dev_conf.rxmode.offloads;
1487                 /* Apply Tx offloads configuration */
1488                 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1489                         port->tx_conf[k].offloads =
1490                                 port->dev_conf.txmode.offloads;
1491
1492                 if (eth_link_speed)
1493                         port->dev_conf.link_speeds = eth_link_speed;
1494
1495                 /* set flag to initialize port/queue */
1496                 port->need_reconfig = 1;
1497                 port->need_reconfig_queues = 1;
1498                 port->tx_metadata = 0;
1499
1500                 /* Check for maximum number of segments per MTU. Accordingly
1501                  * update the mbuf data size.
1502                  */
1503                 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1504                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1505                         data_size = rx_mode.max_rx_pkt_len /
1506                                 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1507
1508                         if ((data_size + RTE_PKTMBUF_HEADROOM) >
1509                                                         mbuf_data_size[0]) {
1510                                 mbuf_data_size[0] = data_size +
1511                                                  RTE_PKTMBUF_HEADROOM;
1512                                 warning = 1;
1513                         }
1514                 }
1515         }
1516
1517         if (warning)
1518                 TESTPMD_LOG(WARNING,
1519                             "Configured mbuf size of the first segment %hu\n",
1520                             mbuf_data_size[0]);
1521         /*
1522          * Create pools of mbuf.
1523          * If NUMA support is disabled, create a single pool of mbuf in
1524          * socket 0 memory by default.
1525          * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1526          *
1527          * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1528          * nb_txd can be configured at run time.
1529          */
1530         if (param_total_num_mbufs)
1531                 nb_mbuf_per_pool = param_total_num_mbufs;
1532         else {
1533                 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1534                         (nb_lcores * mb_mempool_cache) +
1535                         RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1536                 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1537         }
1538
1539         if (numa_support) {
1540                 uint8_t i, j;
1541
1542                 for (i = 0; i < num_sockets; i++)
1543                         for (j = 0; j < mbuf_data_size_n; j++)
1544                                 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1545                                         mbuf_pool_create(mbuf_data_size[j],
1546                                                           nb_mbuf_per_pool,
1547                                                           socket_ids[i], j);
1548         } else {
1549                 uint8_t i;
1550
1551                 for (i = 0; i < mbuf_data_size_n; i++)
1552                         mempools[i] = mbuf_pool_create
1553                                         (mbuf_data_size[i],
1554                                          nb_mbuf_per_pool,
1555                                          socket_num == UMA_NO_CONFIG ?
1556                                          0 : socket_num, i);
1557         }
1558
1559         init_port_config();
1560
1561         gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1562                 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1563         /*
1564          * Records which Mbuf pool to use by each logical core, if needed.
1565          */
1566         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1567                 mbp = mbuf_pool_find(
1568                         rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1569
1570                 if (mbp == NULL)
1571                         mbp = mbuf_pool_find(0, 0);
1572                 fwd_lcores[lc_id]->mbp = mbp;
1573                 /* initialize GSO context */
1574                 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1575                 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1576                 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1577                 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1578                         RTE_ETHER_CRC_LEN;
1579                 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1580         }
1581
1582         /* Configuration of packet forwarding streams. */
1583         if (init_fwd_streams() < 0)
1584                 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1585
1586         fwd_config_setup();
1587
1588         /* create a gro context for each lcore */
1589         gro_param.gro_types = RTE_GRO_TCP_IPV4;
1590         gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1591         gro_param.max_item_per_flow = MAX_PKT_BURST;
1592         for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1593                 gro_param.socket_id = rte_lcore_to_socket_id(
1594                                 fwd_lcores_cpuids[lc_id]);
1595                 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1596                 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1597                         rte_exit(EXIT_FAILURE,
1598                                         "rte_gro_ctx_create() failed\n");
1599                 }
1600         }
1601 }
1602
1603
1604 void
1605 reconfig(portid_t new_port_id, unsigned socket_id)
1606 {
1607         struct rte_port *port;
1608         int ret;
1609
1610         /* Reconfiguration of Ethernet ports. */
1611         port = &ports[new_port_id];
1612
1613         ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1614         if (ret != 0)
1615                 return;
1616
1617         /* set flag to initialize port/queue */
1618         port->need_reconfig = 1;
1619         port->need_reconfig_queues = 1;
1620         port->socket_id = socket_id;
1621
1622         init_port_config();
1623 }
1624
1625
1626 int
1627 init_fwd_streams(void)
1628 {
1629         portid_t pid;
1630         struct rte_port *port;
1631         streamid_t sm_id, nb_fwd_streams_new;
1632         queueid_t q;
1633
1634         /* set socket id according to numa or not */
1635         RTE_ETH_FOREACH_DEV(pid) {
1636                 port = &ports[pid];
1637                 if (nb_rxq > port->dev_info.max_rx_queues) {
1638                         printf("Fail: nb_rxq(%d) is greater than "
1639                                 "max_rx_queues(%d)\n", nb_rxq,
1640                                 port->dev_info.max_rx_queues);
1641                         return -1;
1642                 }
1643                 if (nb_txq > port->dev_info.max_tx_queues) {
1644                         printf("Fail: nb_txq(%d) is greater than "
1645                                 "max_tx_queues(%d)\n", nb_txq,
1646                                 port->dev_info.max_tx_queues);
1647                         return -1;
1648                 }
1649                 if (numa_support) {
1650                         if (port_numa[pid] != NUMA_NO_CONFIG)
1651                                 port->socket_id = port_numa[pid];
1652                         else {
1653                                 port->socket_id = rte_eth_dev_socket_id(pid);
1654
1655                                 /*
1656                                  * if socket_id is invalid,
1657                                  * set to the first available socket.
1658                                  */
1659                                 if (check_socket_id(port->socket_id) < 0)
1660                                         port->socket_id = socket_ids[0];
1661                         }
1662                 }
1663                 else {
1664                         if (socket_num == UMA_NO_CONFIG)
1665                                 port->socket_id = 0;
1666                         else
1667                                 port->socket_id = socket_num;
1668                 }
1669         }
1670
1671         q = RTE_MAX(nb_rxq, nb_txq);
1672         if (q == 0) {
1673                 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1674                 return -1;
1675         }
1676         nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1677         if (nb_fwd_streams_new == nb_fwd_streams)
1678                 return 0;
1679         /* clear the old */
1680         if (fwd_streams != NULL) {
1681                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1682                         if (fwd_streams[sm_id] == NULL)
1683                                 continue;
1684                         rte_free(fwd_streams[sm_id]);
1685                         fwd_streams[sm_id] = NULL;
1686                 }
1687                 rte_free(fwd_streams);
1688                 fwd_streams = NULL;
1689         }
1690
1691         /* init new */
1692         nb_fwd_streams = nb_fwd_streams_new;
1693         if (nb_fwd_streams) {
1694                 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1695                         sizeof(struct fwd_stream *) * nb_fwd_streams,
1696                         RTE_CACHE_LINE_SIZE);
1697                 if (fwd_streams == NULL)
1698                         rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1699                                  " (struct fwd_stream *)) failed\n",
1700                                  nb_fwd_streams);
1701
1702                 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1703                         fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1704                                 " struct fwd_stream", sizeof(struct fwd_stream),
1705                                 RTE_CACHE_LINE_SIZE);
1706                         if (fwd_streams[sm_id] == NULL)
1707                                 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1708                                          "(struct fwd_stream) failed\n");
1709                 }
1710         }
1711
1712         return 0;
1713 }
1714
1715 static void
1716 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1717 {
1718         uint64_t total_burst, sburst;
1719         uint64_t nb_burst;
1720         uint64_t burst_stats[4];
1721         uint16_t pktnb_stats[4];
1722         uint16_t nb_pkt;
1723         int burst_percent[4], sburstp;
1724         int i;
1725
1726         /*
1727          * First compute the total number of packet bursts and the
1728          * two highest numbers of bursts of the same number of packets.
1729          */
1730         memset(&burst_stats, 0x0, sizeof(burst_stats));
1731         memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1732
1733         /* Show stats for 0 burst size always */
1734         total_burst = pbs->pkt_burst_spread[0];
1735         burst_stats[0] = pbs->pkt_burst_spread[0];
1736         pktnb_stats[0] = 0;
1737
1738         /* Find the next 2 burst sizes with highest occurrences. */
1739         for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1740                 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1741
1742                 if (nb_burst == 0)
1743                         continue;
1744
1745                 total_burst += nb_burst;
1746
1747                 if (nb_burst > burst_stats[1]) {
1748                         burst_stats[2] = burst_stats[1];
1749                         pktnb_stats[2] = pktnb_stats[1];
1750                         burst_stats[1] = nb_burst;
1751                         pktnb_stats[1] = nb_pkt;
1752                 } else if (nb_burst > burst_stats[2]) {
1753                         burst_stats[2] = nb_burst;
1754                         pktnb_stats[2] = nb_pkt;
1755                 }
1756         }
1757         if (total_burst == 0)
1758                 return;
1759
1760         printf("  %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1761         for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1762                 if (i == 3) {
1763                         printf("%d%% of other]\n", 100 - sburstp);
1764                         return;
1765                 }
1766
1767                 sburst += burst_stats[i];
1768                 if (sburst == total_burst) {
1769                         printf("%d%% of %d pkts]\n",
1770                                 100 - sburstp, (int) pktnb_stats[i]);
1771                         return;
1772                 }
1773
1774                 burst_percent[i] =
1775                         (double)burst_stats[i] / total_burst * 100;
1776                 printf("%d%% of %d pkts + ",
1777                         burst_percent[i], (int) pktnb_stats[i]);
1778                 sburstp += burst_percent[i];
1779         }
1780 }
1781
1782 static void
1783 fwd_stream_stats_display(streamid_t stream_id)
1784 {
1785         struct fwd_stream *fs;
1786         static const char *fwd_top_stats_border = "-------";
1787
1788         fs = fwd_streams[stream_id];
1789         if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1790             (fs->fwd_dropped == 0))
1791                 return;
1792         printf("\n  %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1793                "TX Port=%2d/Queue=%2d %s\n",
1794                fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1795                fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1796         printf("  RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1797                " TX-dropped: %-14"PRIu64,
1798                fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1799
1800         /* if checksum mode */
1801         if (cur_fwd_eng == &csum_fwd_engine) {
1802                 printf("  RX- bad IP checksum: %-14"PRIu64
1803                        "  Rx- bad L4 checksum: %-14"PRIu64
1804                        " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1805                         fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1806                         fs->rx_bad_outer_l4_csum);
1807                 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1808                         fs->rx_bad_outer_ip_csum);
1809         } else {
1810                 printf("\n");
1811         }
1812
1813         if (record_burst_stats) {
1814                 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1815                 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1816         }
1817 }
1818
1819 void
1820 fwd_stats_display(void)
1821 {
1822         static const char *fwd_stats_border = "----------------------";
1823         static const char *acc_stats_border = "+++++++++++++++";
1824         struct {
1825                 struct fwd_stream *rx_stream;
1826                 struct fwd_stream *tx_stream;
1827                 uint64_t tx_dropped;
1828                 uint64_t rx_bad_ip_csum;
1829                 uint64_t rx_bad_l4_csum;
1830                 uint64_t rx_bad_outer_l4_csum;
1831                 uint64_t rx_bad_outer_ip_csum;
1832         } ports_stats[RTE_MAX_ETHPORTS];
1833         uint64_t total_rx_dropped = 0;
1834         uint64_t total_tx_dropped = 0;
1835         uint64_t total_rx_nombuf = 0;
1836         struct rte_eth_stats stats;
1837         uint64_t fwd_cycles = 0;
1838         uint64_t total_recv = 0;
1839         uint64_t total_xmit = 0;
1840         struct rte_port *port;
1841         streamid_t sm_id;
1842         portid_t pt_id;
1843         int i;
1844
1845         memset(ports_stats, 0, sizeof(ports_stats));
1846
1847         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1848                 struct fwd_stream *fs = fwd_streams[sm_id];
1849
1850                 if (cur_fwd_config.nb_fwd_streams >
1851                     cur_fwd_config.nb_fwd_ports) {
1852                         fwd_stream_stats_display(sm_id);
1853                 } else {
1854                         ports_stats[fs->tx_port].tx_stream = fs;
1855                         ports_stats[fs->rx_port].rx_stream = fs;
1856                 }
1857
1858                 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1859
1860                 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1861                 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1862                 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1863                                 fs->rx_bad_outer_l4_csum;
1864                 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
1865                                 fs->rx_bad_outer_ip_csum;
1866
1867                 if (record_core_cycles)
1868                         fwd_cycles += fs->core_cycles;
1869         }
1870         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1871                 pt_id = fwd_ports_ids[i];
1872                 port = &ports[pt_id];
1873
1874                 rte_eth_stats_get(pt_id, &stats);
1875                 stats.ipackets -= port->stats.ipackets;
1876                 stats.opackets -= port->stats.opackets;
1877                 stats.ibytes -= port->stats.ibytes;
1878                 stats.obytes -= port->stats.obytes;
1879                 stats.imissed -= port->stats.imissed;
1880                 stats.oerrors -= port->stats.oerrors;
1881                 stats.rx_nombuf -= port->stats.rx_nombuf;
1882
1883                 total_recv += stats.ipackets;
1884                 total_xmit += stats.opackets;
1885                 total_rx_dropped += stats.imissed;
1886                 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1887                 total_tx_dropped += stats.oerrors;
1888                 total_rx_nombuf  += stats.rx_nombuf;
1889
1890                 printf("\n  %s Forward statistics for port %-2d %s\n",
1891                        fwd_stats_border, pt_id, fwd_stats_border);
1892
1893                 printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1894                        "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1895                        stats.ipackets + stats.imissed);
1896
1897                 if (cur_fwd_eng == &csum_fwd_engine) {
1898                         printf("  Bad-ipcsum: %-14"PRIu64
1899                                " Bad-l4csum: %-14"PRIu64
1900                                "Bad-outer-l4csum: %-14"PRIu64"\n",
1901                                ports_stats[pt_id].rx_bad_ip_csum,
1902                                ports_stats[pt_id].rx_bad_l4_csum,
1903                                ports_stats[pt_id].rx_bad_outer_l4_csum);
1904                         printf("  Bad-outer-ipcsum: %-14"PRIu64"\n",
1905                                ports_stats[pt_id].rx_bad_outer_ip_csum);
1906                 }
1907                 if (stats.ierrors + stats.rx_nombuf > 0) {
1908                         printf("  RX-error: %-"PRIu64"\n", stats.ierrors);
1909                         printf("  RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1910                 }
1911
1912                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1913                        "TX-total: %-"PRIu64"\n",
1914                        stats.opackets, ports_stats[pt_id].tx_dropped,
1915                        stats.opackets + ports_stats[pt_id].tx_dropped);
1916
1917                 if (record_burst_stats) {
1918                         if (ports_stats[pt_id].rx_stream)
1919                                 pkt_burst_stats_display("RX",
1920                                         &ports_stats[pt_id].rx_stream->rx_burst_stats);
1921                         if (ports_stats[pt_id].tx_stream)
1922                                 pkt_burst_stats_display("TX",
1923                                 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1924                 }
1925
1926                 printf("  %s--------------------------------%s\n",
1927                        fwd_stats_border, fwd_stats_border);
1928         }
1929
1930         printf("\n  %s Accumulated forward statistics for all ports"
1931                "%s\n",
1932                acc_stats_border, acc_stats_border);
1933         printf("  RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1934                "%-"PRIu64"\n"
1935                "  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1936                "%-"PRIu64"\n",
1937                total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1938                total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1939         if (total_rx_nombuf > 0)
1940                 printf("  RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1941         printf("  %s++++++++++++++++++++++++++++++++++++++++++++++"
1942                "%s\n",
1943                acc_stats_border, acc_stats_border);
1944         if (record_core_cycles) {
1945 #define CYC_PER_MHZ 1E6
1946                 if (total_recv > 0 || total_xmit > 0) {
1947                         uint64_t total_pkts = 0;
1948                         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1949                             strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1950                                 total_pkts = total_xmit;
1951                         else
1952                                 total_pkts = total_recv;
1953
1954                         printf("\n  CPU cycles/packet=%.2F (total cycles="
1955                                "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1956                                " MHz Clock\n",
1957                                (double) fwd_cycles / total_pkts,
1958                                fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1959                                (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1960                 }
1961         }
1962 }
1963
1964 void
1965 fwd_stats_reset(void)
1966 {
1967         streamid_t sm_id;
1968         portid_t pt_id;
1969         int i;
1970
1971         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1972                 pt_id = fwd_ports_ids[i];
1973                 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1974         }
1975         for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1976                 struct fwd_stream *fs = fwd_streams[sm_id];
1977
1978                 fs->rx_packets = 0;
1979                 fs->tx_packets = 0;
1980                 fs->fwd_dropped = 0;
1981                 fs->rx_bad_ip_csum = 0;
1982                 fs->rx_bad_l4_csum = 0;
1983                 fs->rx_bad_outer_l4_csum = 0;
1984                 fs->rx_bad_outer_ip_csum = 0;
1985
1986                 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1987                 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1988                 fs->core_cycles = 0;
1989         }
1990 }
1991
1992 static void
1993 flush_fwd_rx_queues(void)
1994 {
1995         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1996         portid_t  rxp;
1997         portid_t port_id;
1998         queueid_t rxq;
1999         uint16_t  nb_rx;
2000         uint16_t  i;
2001         uint8_t   j;
2002         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2003         uint64_t timer_period;
2004
2005         /* convert to number of cycles */
2006         timer_period = rte_get_timer_hz(); /* 1 second timeout */
2007
2008         for (j = 0; j < 2; j++) {
2009                 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2010                         for (rxq = 0; rxq < nb_rxq; rxq++) {
2011                                 port_id = fwd_ports_ids[rxp];
2012                                 /**
2013                                 * testpmd can stuck in the below do while loop
2014                                 * if rte_eth_rx_burst() always returns nonzero
2015                                 * packets. So timer is added to exit this loop
2016                                 * after 1sec timer expiry.
2017                                 */
2018                                 prev_tsc = rte_rdtsc();
2019                                 do {
2020                                         nb_rx = rte_eth_rx_burst(port_id, rxq,
2021                                                 pkts_burst, MAX_PKT_BURST);
2022                                         for (i = 0; i < nb_rx; i++)
2023                                                 rte_pktmbuf_free(pkts_burst[i]);
2024
2025                                         cur_tsc = rte_rdtsc();
2026                                         diff_tsc = cur_tsc - prev_tsc;
2027                                         timer_tsc += diff_tsc;
2028                                 } while ((nb_rx > 0) &&
2029                                         (timer_tsc < timer_period));
2030                                 timer_tsc = 0;
2031                         }
2032                 }
2033                 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2034         }
2035 }
2036
2037 static void
2038 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2039 {
2040         struct fwd_stream **fsm;
2041         streamid_t nb_fs;
2042         streamid_t sm_id;
2043 #ifdef RTE_LIB_BITRATESTATS
2044         uint64_t tics_per_1sec;
2045         uint64_t tics_datum;
2046         uint64_t tics_current;
2047         uint16_t i, cnt_ports;
2048
2049         cnt_ports = nb_ports;
2050         tics_datum = rte_rdtsc();
2051         tics_per_1sec = rte_get_timer_hz();
2052 #endif
2053         fsm = &fwd_streams[fc->stream_idx];
2054         nb_fs = fc->stream_nb;
2055         do {
2056                 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2057                         (*pkt_fwd)(fsm[sm_id]);
2058 #ifdef RTE_LIB_BITRATESTATS
2059                 if (bitrate_enabled != 0 &&
2060                                 bitrate_lcore_id == rte_lcore_id()) {
2061                         tics_current = rte_rdtsc();
2062                         if (tics_current - tics_datum >= tics_per_1sec) {
2063                                 /* Periodic bitrate calculation */
2064                                 for (i = 0; i < cnt_ports; i++)
2065                                         rte_stats_bitrate_calc(bitrate_data,
2066                                                 ports_ids[i]);
2067                                 tics_datum = tics_current;
2068                         }
2069                 }
2070 #endif
2071 #ifdef RTE_LIB_LATENCYSTATS
2072                 if (latencystats_enabled != 0 &&
2073                                 latencystats_lcore_id == rte_lcore_id())
2074                         rte_latencystats_update();
2075 #endif
2076
2077         } while (! fc->stopped);
2078 }
2079
2080 static int
2081 start_pkt_forward_on_core(void *fwd_arg)
2082 {
2083         run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2084                              cur_fwd_config.fwd_eng->packet_fwd);
2085         return 0;
2086 }
2087
2088 /*
2089  * Run the TXONLY packet forwarding engine to send a single burst of packets.
2090  * Used to start communication flows in network loopback test configurations.
2091  */
2092 static int
2093 run_one_txonly_burst_on_core(void *fwd_arg)
2094 {
2095         struct fwd_lcore *fwd_lc;
2096         struct fwd_lcore tmp_lcore;
2097
2098         fwd_lc = (struct fwd_lcore *) fwd_arg;
2099         tmp_lcore = *fwd_lc;
2100         tmp_lcore.stopped = 1;
2101         run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2102         return 0;
2103 }
2104
2105 /*
2106  * Launch packet forwarding:
2107  *     - Setup per-port forwarding context.
2108  *     - launch logical cores with their forwarding configuration.
2109  */
2110 static void
2111 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2112 {
2113         port_fwd_begin_t port_fwd_begin;
2114         unsigned int i;
2115         unsigned int lc_id;
2116         int diag;
2117
2118         port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2119         if (port_fwd_begin != NULL) {
2120                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2121                         (*port_fwd_begin)(fwd_ports_ids[i]);
2122         }
2123         for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2124                 lc_id = fwd_lcores_cpuids[i];
2125                 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2126                         fwd_lcores[i]->stopped = 0;
2127                         diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2128                                                      fwd_lcores[i], lc_id);
2129                         if (diag != 0)
2130                                 printf("launch lcore %u failed - diag=%d\n",
2131                                        lc_id, diag);
2132                 }
2133         }
2134 }
2135
2136 /*
2137  * Launch packet forwarding configuration.
2138  */
2139 void
2140 start_packet_forwarding(int with_tx_first)
2141 {
2142         port_fwd_begin_t port_fwd_begin;
2143         port_fwd_end_t  port_fwd_end;
2144         struct rte_port *port;
2145         unsigned int i;
2146         portid_t   pt_id;
2147
2148         if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2149                 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2150
2151         if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2152                 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2153
2154         if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2155                 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2156                 (!nb_rxq || !nb_txq))
2157                 rte_exit(EXIT_FAILURE,
2158                         "Either rxq or txq are 0, cannot use %s fwd mode\n",
2159                         cur_fwd_eng->fwd_mode_name);
2160
2161         if (all_ports_started() == 0) {
2162                 printf("Not all ports were started\n");
2163                 return;
2164         }
2165         if (test_done == 0) {
2166                 printf("Packet forwarding already started\n");
2167                 return;
2168         }
2169
2170
2171         if(dcb_test) {
2172                 for (i = 0; i < nb_fwd_ports; i++) {
2173                         pt_id = fwd_ports_ids[i];
2174                         port = &ports[pt_id];
2175                         if (!port->dcb_flag) {
2176                                 printf("In DCB mode, all forwarding ports must "
2177                                        "be configured in this mode.\n");
2178                                 return;
2179                         }
2180                 }
2181                 if (nb_fwd_lcores == 1) {
2182                         printf("In DCB mode,the nb forwarding cores "
2183                                "should be larger than 1.\n");
2184                         return;
2185                 }
2186         }
2187         test_done = 0;
2188
2189         fwd_config_setup();
2190
2191         if(!no_flush_rx)
2192                 flush_fwd_rx_queues();
2193
2194         pkt_fwd_config_display(&cur_fwd_config);
2195         rxtx_config_display();
2196
2197         fwd_stats_reset();
2198         if (with_tx_first) {
2199                 port_fwd_begin = tx_only_engine.port_fwd_begin;
2200                 if (port_fwd_begin != NULL) {
2201                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2202                                 (*port_fwd_begin)(fwd_ports_ids[i]);
2203                 }
2204                 while (with_tx_first--) {
2205                         launch_packet_forwarding(
2206                                         run_one_txonly_burst_on_core);
2207                         rte_eal_mp_wait_lcore();
2208                 }
2209                 port_fwd_end = tx_only_engine.port_fwd_end;
2210                 if (port_fwd_end != NULL) {
2211                         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2212                                 (*port_fwd_end)(fwd_ports_ids[i]);
2213                 }
2214         }
2215         launch_packet_forwarding(start_pkt_forward_on_core);
2216 }
2217
2218 void
2219 stop_packet_forwarding(void)
2220 {
2221         port_fwd_end_t port_fwd_end;
2222         lcoreid_t lc_id;
2223         portid_t pt_id;
2224         int i;
2225
2226         if (test_done) {
2227                 printf("Packet forwarding not started\n");
2228                 return;
2229         }
2230         printf("Telling cores to stop...");
2231         for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2232                 fwd_lcores[lc_id]->stopped = 1;
2233         printf("\nWaiting for lcores to finish...\n");
2234         rte_eal_mp_wait_lcore();
2235         port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2236         if (port_fwd_end != NULL) {
2237                 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2238                         pt_id = fwd_ports_ids[i];
2239                         (*port_fwd_end)(pt_id);
2240                 }
2241         }
2242
2243         fwd_stats_display();
2244
2245         printf("\nDone.\n");
2246         test_done = 1;
2247 }
2248
2249 void
2250 dev_set_link_up(portid_t pid)
2251 {
2252         if (rte_eth_dev_set_link_up(pid) < 0)
2253                 printf("\nSet link up fail.\n");
2254 }
2255
2256 void
2257 dev_set_link_down(portid_t pid)
2258 {
2259         if (rte_eth_dev_set_link_down(pid) < 0)
2260                 printf("\nSet link down fail.\n");
2261 }
2262
2263 static int
2264 all_ports_started(void)
2265 {
2266         portid_t pi;
2267         struct rte_port *port;
2268
2269         RTE_ETH_FOREACH_DEV(pi) {
2270                 port = &ports[pi];
2271                 /* Check if there is a port which is not started */
2272                 if ((port->port_status != RTE_PORT_STARTED) &&
2273                         (port->slave_flag == 0))
2274                         return 0;
2275         }
2276
2277         /* No port is not started */
2278         return 1;
2279 }
2280
2281 int
2282 port_is_stopped(portid_t port_id)
2283 {
2284         struct rte_port *port = &ports[port_id];
2285
2286         if ((port->port_status != RTE_PORT_STOPPED) &&
2287             (port->slave_flag == 0))
2288                 return 0;
2289         return 1;
2290 }
2291
2292 int
2293 all_ports_stopped(void)
2294 {
2295         portid_t pi;
2296
2297         RTE_ETH_FOREACH_DEV(pi) {
2298                 if (!port_is_stopped(pi))
2299                         return 0;
2300         }
2301
2302         return 1;
2303 }
2304
2305 int
2306 port_is_started(portid_t port_id)
2307 {
2308         if (port_id_is_invalid(port_id, ENABLED_WARN))
2309                 return 0;
2310
2311         if (ports[port_id].port_status != RTE_PORT_STARTED)
2312                 return 0;
2313
2314         return 1;
2315 }
2316
2317 /* Configure the Rx and Tx hairpin queues for the selected port. */
2318 static int
2319 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2320 {
2321         queueid_t qi;
2322         struct rte_eth_hairpin_conf hairpin_conf = {
2323                 .peer_count = 1,
2324         };
2325         int i;
2326         int diag;
2327         struct rte_port *port = &ports[pi];
2328         uint16_t peer_rx_port = pi;
2329         uint16_t peer_tx_port = pi;
2330         uint32_t manual = 1;
2331         uint32_t tx_exp = hairpin_mode & 0x10;
2332
2333         if (!(hairpin_mode & 0xf)) {
2334                 peer_rx_port = pi;
2335                 peer_tx_port = pi;
2336                 manual = 0;
2337         } else if (hairpin_mode & 0x1) {
2338                 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2339                                                        RTE_ETH_DEV_NO_OWNER);
2340                 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2341                         peer_tx_port = rte_eth_find_next_owned_by(0,
2342                                                 RTE_ETH_DEV_NO_OWNER);
2343                 if (p_pi != RTE_MAX_ETHPORTS) {
2344                         peer_rx_port = p_pi;
2345                 } else {
2346                         uint16_t next_pi;
2347
2348                         /* Last port will be the peer RX port of the first. */
2349                         RTE_ETH_FOREACH_DEV(next_pi)
2350                                 peer_rx_port = next_pi;
2351                 }
2352                 manual = 1;
2353         } else if (hairpin_mode & 0x2) {
2354                 if (cnt_pi & 0x1) {
2355                         peer_rx_port = p_pi;
2356                 } else {
2357                         peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2358                                                 RTE_ETH_DEV_NO_OWNER);
2359                         if (peer_rx_port >= RTE_MAX_ETHPORTS)
2360                                 peer_rx_port = pi;
2361                 }
2362                 peer_tx_port = peer_rx_port;
2363                 manual = 1;
2364         }
2365
2366         for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2367                 hairpin_conf.peers[0].port = peer_rx_port;
2368                 hairpin_conf.peers[0].queue = i + nb_rxq;
2369                 hairpin_conf.manual_bind = !!manual;
2370                 hairpin_conf.tx_explicit = !!tx_exp;
2371                 diag = rte_eth_tx_hairpin_queue_setup
2372                         (pi, qi, nb_txd, &hairpin_conf);
2373                 i++;
2374                 if (diag == 0)
2375                         continue;
2376
2377                 /* Fail to setup rx queue, return */
2378                 if (rte_atomic16_cmpset(&(port->port_status),
2379                                         RTE_PORT_HANDLING,
2380                                         RTE_PORT_STOPPED) == 0)
2381                         printf("Port %d can not be set back "
2382                                         "to stopped\n", pi);
2383                 printf("Fail to configure port %d hairpin "
2384                                 "queues\n", pi);
2385                 /* try to reconfigure queues next time */
2386                 port->need_reconfig_queues = 1;
2387                 return -1;
2388         }
2389         for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2390                 hairpin_conf.peers[0].port = peer_tx_port;
2391                 hairpin_conf.peers[0].queue = i + nb_txq;
2392                 hairpin_conf.manual_bind = !!manual;
2393                 hairpin_conf.tx_explicit = !!tx_exp;
2394                 diag = rte_eth_rx_hairpin_queue_setup
2395                         (pi, qi, nb_rxd, &hairpin_conf);
2396                 i++;
2397                 if (diag == 0)
2398                         continue;
2399
2400                 /* Fail to setup rx queue, return */
2401                 if (rte_atomic16_cmpset(&(port->port_status),
2402                                         RTE_PORT_HANDLING,
2403                                         RTE_PORT_STOPPED) == 0)
2404                         printf("Port %d can not be set back "
2405                                         "to stopped\n", pi);
2406                 printf("Fail to configure port %d hairpin "
2407                                 "queues\n", pi);
2408                 /* try to reconfigure queues next time */
2409                 port->need_reconfig_queues = 1;
2410                 return -1;
2411         }
2412         return 0;
2413 }
2414
2415 /* Configure the Rx with optional split. */
2416 int
2417 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2418                uint16_t nb_rx_desc, unsigned int socket_id,
2419                struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2420 {
2421         union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2422         unsigned int i, mp_n;
2423         int ret;
2424
2425         if (rx_pkt_nb_segs <= 1 ||
2426             (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2427                 rx_conf->rx_seg = NULL;
2428                 rx_conf->rx_nseg = 0;
2429                 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2430                                              nb_rx_desc, socket_id,
2431                                              rx_conf, mp);
2432                 return ret;
2433         }
2434         for (i = 0; i < rx_pkt_nb_segs; i++) {
2435                 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2436                 struct rte_mempool *mpx;
2437                 /*
2438                  * Use last valid pool for the segments with number
2439                  * exceeding the pool index.
2440                  */
2441                 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2442                 mpx = mbuf_pool_find(socket_id, mp_n);
2443                 /* Handle zero as mbuf data buffer size. */
2444                 rx_seg->length = rx_pkt_seg_lengths[i] ?
2445                                    rx_pkt_seg_lengths[i] :
2446                                    mbuf_data_size[mp_n];
2447                 rx_seg->offset = i < rx_pkt_nb_offs ?
2448                                    rx_pkt_seg_offsets[i] : 0;
2449                 rx_seg->mp = mpx ? mpx : mp;
2450         }
2451         rx_conf->rx_nseg = rx_pkt_nb_segs;
2452         rx_conf->rx_seg = rx_useg;
2453         ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2454                                     socket_id, rx_conf, NULL);
2455         rx_conf->rx_seg = NULL;
2456         rx_conf->rx_nseg = 0;
2457         return ret;
2458 }
2459
2460 int
2461 start_port(portid_t pid)
2462 {
2463         int diag, need_check_link_status = -1;
2464         portid_t pi;
2465         portid_t p_pi = RTE_MAX_ETHPORTS;
2466         portid_t pl[RTE_MAX_ETHPORTS];
2467         portid_t peer_pl[RTE_MAX_ETHPORTS];
2468         uint16_t cnt_pi = 0;
2469         uint16_t cfg_pi = 0;
2470         int peer_pi;
2471         queueid_t qi;
2472         struct rte_port *port;
2473         struct rte_ether_addr mac_addr;
2474         struct rte_eth_hairpin_cap cap;
2475
2476         if (port_id_is_invalid(pid, ENABLED_WARN))
2477                 return 0;
2478
2479         if(dcb_config)
2480                 dcb_test = 1;
2481         RTE_ETH_FOREACH_DEV(pi) {
2482                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2483                         continue;
2484
2485                 need_check_link_status = 0;
2486                 port = &ports[pi];
2487                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2488                                                  RTE_PORT_HANDLING) == 0) {
2489                         printf("Port %d is now not stopped\n", pi);
2490                         continue;
2491                 }
2492
2493                 if (port->need_reconfig > 0) {
2494                         port->need_reconfig = 0;
2495
2496                         if (flow_isolate_all) {
2497                                 int ret = port_flow_isolate(pi, 1);
2498                                 if (ret) {
2499                                         printf("Failed to apply isolated"
2500                                                " mode on port %d\n", pi);
2501                                         return -1;
2502                                 }
2503                         }
2504                         configure_rxtx_dump_callbacks(0);
2505                         printf("Configuring Port %d (socket %u)\n", pi,
2506                                         port->socket_id);
2507                         if (nb_hairpinq > 0 &&
2508                             rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2509                                 printf("Port %d doesn't support hairpin "
2510                                        "queues\n", pi);
2511                                 return -1;
2512                         }
2513                         /* configure port */
2514                         diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2515                                                      nb_txq + nb_hairpinq,
2516                                                      &(port->dev_conf));
2517                         if (diag != 0) {
2518                                 if (rte_atomic16_cmpset(&(port->port_status),
2519                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2520                                         printf("Port %d can not be set back "
2521                                                         "to stopped\n", pi);
2522                                 printf("Fail to configure port %d\n", pi);
2523                                 /* try to reconfigure port next time */
2524                                 port->need_reconfig = 1;
2525                                 return -1;
2526                         }
2527                 }
2528                 if (port->need_reconfig_queues > 0) {
2529                         port->need_reconfig_queues = 0;
2530                         /* setup tx queues */
2531                         for (qi = 0; qi < nb_txq; qi++) {
2532                                 if ((numa_support) &&
2533                                         (txring_numa[pi] != NUMA_NO_CONFIG))
2534                                         diag = rte_eth_tx_queue_setup(pi, qi,
2535                                                 port->nb_tx_desc[qi],
2536                                                 txring_numa[pi],
2537                                                 &(port->tx_conf[qi]));
2538                                 else
2539                                         diag = rte_eth_tx_queue_setup(pi, qi,
2540                                                 port->nb_tx_desc[qi],
2541                                                 port->socket_id,
2542                                                 &(port->tx_conf[qi]));
2543
2544                                 if (diag == 0)
2545                                         continue;
2546
2547                                 /* Fail to setup tx queue, return */
2548                                 if (rte_atomic16_cmpset(&(port->port_status),
2549                                                         RTE_PORT_HANDLING,
2550                                                         RTE_PORT_STOPPED) == 0)
2551                                         printf("Port %d can not be set back "
2552                                                         "to stopped\n", pi);
2553                                 printf("Fail to configure port %d tx queues\n",
2554                                        pi);
2555                                 /* try to reconfigure queues next time */
2556                                 port->need_reconfig_queues = 1;
2557                                 return -1;
2558                         }
2559                         for (qi = 0; qi < nb_rxq; qi++) {
2560                                 /* setup rx queues */
2561                                 if ((numa_support) &&
2562                                         (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2563                                         struct rte_mempool * mp =
2564                                                 mbuf_pool_find
2565                                                         (rxring_numa[pi], 0);
2566                                         if (mp == NULL) {
2567                                                 printf("Failed to setup RX queue:"
2568                                                         "No mempool allocation"
2569                                                         " on the socket %d\n",
2570                                                         rxring_numa[pi]);
2571                                                 return -1;
2572                                         }
2573
2574                                         diag = rx_queue_setup(pi, qi,
2575                                              port->nb_rx_desc[qi],
2576                                              rxring_numa[pi],
2577                                              &(port->rx_conf[qi]),
2578                                              mp);
2579                                 } else {
2580                                         struct rte_mempool *mp =
2581                                                 mbuf_pool_find
2582                                                         (port->socket_id, 0);
2583                                         if (mp == NULL) {
2584                                                 printf("Failed to setup RX queue:"
2585                                                         "No mempool allocation"
2586                                                         " on the socket %d\n",
2587                                                         port->socket_id);
2588                                                 return -1;
2589                                         }
2590                                         diag = rx_queue_setup(pi, qi,
2591                                              port->nb_rx_desc[qi],
2592                                              port->socket_id,
2593                                              &(port->rx_conf[qi]),
2594                                              mp);
2595                                 }
2596                                 if (diag == 0)
2597                                         continue;
2598
2599                                 /* Fail to setup rx queue, return */
2600                                 if (rte_atomic16_cmpset(&(port->port_status),
2601                                                         RTE_PORT_HANDLING,
2602                                                         RTE_PORT_STOPPED) == 0)
2603                                         printf("Port %d can not be set back "
2604                                                         "to stopped\n", pi);
2605                                 printf("Fail to configure port %d rx queues\n",
2606                                        pi);
2607                                 /* try to reconfigure queues next time */
2608                                 port->need_reconfig_queues = 1;
2609                                 return -1;
2610                         }
2611                         /* setup hairpin queues */
2612                         if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2613                                 return -1;
2614                 }
2615                 configure_rxtx_dump_callbacks(verbose_level);
2616                 if (clear_ptypes) {
2617                         diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2618                                         NULL, 0);
2619                         if (diag < 0)
2620                                 printf(
2621                                 "Port %d: Failed to disable Ptype parsing\n",
2622                                 pi);
2623                 }
2624
2625                 p_pi = pi;
2626                 cnt_pi++;
2627
2628                 /* start port */
2629                 diag = rte_eth_dev_start(pi);
2630                 if (diag < 0) {
2631                         printf("Fail to start port %d: %s\n", pi,
2632                                rte_strerror(-diag));
2633
2634                         /* Fail to setup rx queue, return */
2635                         if (rte_atomic16_cmpset(&(port->port_status),
2636                                 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2637                                 printf("Port %d can not be set back to "
2638                                                         "stopped\n", pi);
2639                         continue;
2640                 }
2641
2642                 if (rte_atomic16_cmpset(&(port->port_status),
2643                         RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2644                         printf("Port %d can not be set into started\n", pi);
2645
2646                 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2647                         printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2648                                 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2649                                 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2650                                 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2651
2652                 /* at least one port started, need checking link status */
2653                 need_check_link_status = 1;
2654
2655                 pl[cfg_pi++] = pi;
2656         }
2657
2658         if (need_check_link_status == 1 && !no_link_check)
2659                 check_all_ports_link_status(RTE_PORT_ALL);
2660         else if (need_check_link_status == 0)
2661                 printf("Please stop the ports first\n");
2662
2663         if (hairpin_mode & 0xf) {
2664                 uint16_t i;
2665                 int j;
2666
2667                 /* bind all started hairpin ports */
2668                 for (i = 0; i < cfg_pi; i++) {
2669                         pi = pl[i];
2670                         /* bind current Tx to all peer Rx */
2671                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2672                                                         RTE_MAX_ETHPORTS, 1);
2673                         if (peer_pi < 0)
2674                                 return peer_pi;
2675                         for (j = 0; j < peer_pi; j++) {
2676                                 if (!port_is_started(peer_pl[j]))
2677                                         continue;
2678                                 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2679                                 if (diag < 0) {
2680                                         printf("Error during binding hairpin"
2681                                                " Tx port %u to %u: %s\n",
2682                                                pi, peer_pl[j],
2683                                                rte_strerror(-diag));
2684                                         return -1;
2685                                 }
2686                         }
2687                         /* bind all peer Tx to current Rx */
2688                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2689                                                         RTE_MAX_ETHPORTS, 0);
2690                         if (peer_pi < 0)
2691                                 return peer_pi;
2692                         for (j = 0; j < peer_pi; j++) {
2693                                 if (!port_is_started(peer_pl[j]))
2694                                         continue;
2695                                 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2696                                 if (diag < 0) {
2697                                         printf("Error during binding hairpin"
2698                                                " Tx port %u to %u: %s\n",
2699                                                peer_pl[j], pi,
2700                                                rte_strerror(-diag));
2701                                         return -1;
2702                                 }
2703                         }
2704                 }
2705         }
2706
2707         printf("Done\n");
2708         return 0;
2709 }
2710
2711 void
2712 stop_port(portid_t pid)
2713 {
2714         portid_t pi;
2715         struct rte_port *port;
2716         int need_check_link_status = 0;
2717         portid_t peer_pl[RTE_MAX_ETHPORTS];
2718         int peer_pi;
2719
2720         if (dcb_test) {
2721                 dcb_test = 0;
2722                 dcb_config = 0;
2723         }
2724
2725         if (port_id_is_invalid(pid, ENABLED_WARN))
2726                 return;
2727
2728         printf("Stopping ports...\n");
2729
2730         RTE_ETH_FOREACH_DEV(pi) {
2731                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2732                         continue;
2733
2734                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2735                         printf("Please remove port %d from forwarding configuration.\n", pi);
2736                         continue;
2737                 }
2738
2739                 if (port_is_bonding_slave(pi)) {
2740                         printf("Please remove port %d from bonded device.\n", pi);
2741                         continue;
2742                 }
2743
2744                 port = &ports[pi];
2745                 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2746                                                 RTE_PORT_HANDLING) == 0)
2747                         continue;
2748
2749                 if (hairpin_mode & 0xf) {
2750                         int j;
2751
2752                         rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2753                         /* unbind all peer Tx from current Rx */
2754                         peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2755                                                         RTE_MAX_ETHPORTS, 0);
2756                         if (peer_pi < 0)
2757                                 continue;
2758                         for (j = 0; j < peer_pi; j++) {
2759                                 if (!port_is_started(peer_pl[j]))
2760                                         continue;
2761                                 rte_eth_hairpin_unbind(peer_pl[j], pi);
2762                         }
2763                 }
2764
2765                 if (port->flow_list)
2766                         port_flow_flush(pi);
2767
2768                 if (rte_eth_dev_stop(pi) != 0)
2769                         RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2770                                 pi);
2771
2772                 if (rte_atomic16_cmpset(&(port->port_status),
2773                         RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2774                         printf("Port %d can not be set into stopped\n", pi);
2775                 need_check_link_status = 1;
2776         }
2777         if (need_check_link_status && !no_link_check)
2778                 check_all_ports_link_status(RTE_PORT_ALL);
2779
2780         printf("Done\n");
2781 }
2782
2783 static void
2784 remove_invalid_ports_in(portid_t *array, portid_t *total)
2785 {
2786         portid_t i;
2787         portid_t new_total = 0;
2788
2789         for (i = 0; i < *total; i++)
2790                 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2791                         array[new_total] = array[i];
2792                         new_total++;
2793                 }
2794         *total = new_total;
2795 }
2796
2797 static void
2798 remove_invalid_ports(void)
2799 {
2800         remove_invalid_ports_in(ports_ids, &nb_ports);
2801         remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2802         nb_cfg_ports = nb_fwd_ports;
2803 }
2804
2805 void
2806 close_port(portid_t pid)
2807 {
2808         portid_t pi;
2809         struct rte_port *port;
2810
2811         if (port_id_is_invalid(pid, ENABLED_WARN))
2812                 return;
2813
2814         printf("Closing ports...\n");
2815
2816         RTE_ETH_FOREACH_DEV(pi) {
2817                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2818                         continue;
2819
2820                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2821                         printf("Please remove port %d from forwarding configuration.\n", pi);
2822                         continue;
2823                 }
2824
2825                 if (port_is_bonding_slave(pi)) {
2826                         printf("Please remove port %d from bonded device.\n", pi);
2827                         continue;
2828                 }
2829
2830                 port = &ports[pi];
2831                 if (rte_atomic16_cmpset(&(port->port_status),
2832                         RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2833                         printf("Port %d is already closed\n", pi);
2834                         continue;
2835                 }
2836
2837                 port_flow_flush(pi);
2838                 rte_eth_dev_close(pi);
2839         }
2840
2841         remove_invalid_ports();
2842         printf("Done\n");
2843 }
2844
2845 void
2846 reset_port(portid_t pid)
2847 {
2848         int diag;
2849         portid_t pi;
2850         struct rte_port *port;
2851
2852         if (port_id_is_invalid(pid, ENABLED_WARN))
2853                 return;
2854
2855         if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2856                 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2857                 printf("Can not reset port(s), please stop port(s) first.\n");
2858                 return;
2859         }
2860
2861         printf("Resetting ports...\n");
2862
2863         RTE_ETH_FOREACH_DEV(pi) {
2864                 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2865                         continue;
2866
2867                 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2868                         printf("Please remove port %d from forwarding "
2869                                "configuration.\n", pi);
2870                         continue;
2871                 }
2872
2873                 if (port_is_bonding_slave(pi)) {
2874                         printf("Please remove port %d from bonded device.\n",
2875                                pi);
2876                         continue;
2877                 }
2878
2879                 diag = rte_eth_dev_reset(pi);
2880                 if (diag == 0) {
2881                         port = &ports[pi];
2882                         port->need_reconfig = 1;
2883                         port->need_reconfig_queues = 1;
2884                 } else {
2885                         printf("Failed to reset port %d. diag=%d\n", pi, diag);
2886                 }
2887         }
2888
2889         printf("Done\n");
2890 }
2891
2892 void
2893 attach_port(char *identifier)
2894 {
2895         portid_t pi;
2896         struct rte_dev_iterator iterator;
2897
2898         printf("Attaching a new port...\n");
2899
2900         if (identifier == NULL) {
2901                 printf("Invalid parameters are specified\n");
2902                 return;
2903         }
2904
2905         if (rte_dev_probe(identifier) < 0) {
2906                 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2907                 return;
2908         }
2909
2910         /* first attach mode: event */
2911         if (setup_on_probe_event) {
2912                 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2913                 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2914                         if (ports[pi].port_status == RTE_PORT_HANDLING &&
2915                                         ports[pi].need_setup != 0)
2916                                 setup_attached_port(pi);
2917                 return;
2918         }
2919
2920         /* second attach mode: iterator */
2921         RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2922                 /* setup ports matching the devargs used for probing */
2923                 if (port_is_forwarding(pi))
2924                         continue; /* port was already attached before */
2925                 setup_attached_port(pi);
2926         }
2927 }
2928
2929 static void
2930 setup_attached_port(portid_t pi)
2931 {
2932         unsigned int socket_id;
2933         int ret;
2934
2935         socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2936         /* if socket_id is invalid, set to the first available socket. */
2937         if (check_socket_id(socket_id) < 0)
2938                 socket_id = socket_ids[0];
2939         reconfig(pi, socket_id);
2940         ret = rte_eth_promiscuous_enable(pi);
2941         if (ret != 0)
2942                 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2943                         pi, rte_strerror(-ret));
2944
2945         ports_ids[nb_ports++] = pi;
2946         fwd_ports_ids[nb_fwd_ports++] = pi;
2947         nb_cfg_ports = nb_fwd_ports;
2948         ports[pi].need_setup = 0;
2949         ports[pi].port_status = RTE_PORT_STOPPED;
2950
2951         printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2952         printf("Done\n");
2953 }
2954
2955 static void
2956 detach_device(struct rte_device *dev)
2957 {
2958         portid_t sibling;
2959
2960         if (dev == NULL) {
2961                 printf("Device already removed\n");
2962                 return;
2963         }
2964
2965         printf("Removing a device...\n");
2966
2967         RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2968                 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2969                         if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2970                                 printf("Port %u not stopped\n", sibling);
2971                                 return;
2972                         }
2973                         port_flow_flush(sibling);
2974                 }
2975         }
2976
2977         if (rte_dev_remove(dev) < 0) {
2978                 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2979                 return;
2980         }
2981         remove_invalid_ports();
2982
2983         printf("Device is detached\n");
2984         printf("Now total ports is %d\n", nb_ports);
2985         printf("Done\n");
2986         return;
2987 }
2988
2989 void
2990 detach_port_device(portid_t port_id)
2991 {
2992         if (port_id_is_invalid(port_id, ENABLED_WARN))
2993                 return;
2994
2995         if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2996                 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2997                         printf("Port not stopped\n");
2998                         return;
2999                 }
3000                 printf("Port was not closed\n");
3001         }
3002
3003         detach_device(rte_eth_devices[port_id].device);
3004 }
3005
3006 void
3007 detach_devargs(char *identifier)
3008 {
3009         struct rte_dev_iterator iterator;
3010         struct rte_devargs da;
3011         portid_t port_id;
3012
3013         printf("Removing a device...\n");
3014
3015         memset(&da, 0, sizeof(da));
3016         if (rte_devargs_parsef(&da, "%s", identifier)) {
3017                 printf("cannot parse identifier\n");
3018                 if (da.args)
3019                         free(da.args);
3020                 return;
3021         }
3022
3023         RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3024                 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3025                         if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3026                                 printf("Port %u not stopped\n", port_id);
3027                                 rte_eth_iterator_cleanup(&iterator);
3028                                 return;
3029                         }
3030                         port_flow_flush(port_id);
3031                 }
3032         }
3033
3034         if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3035                 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3036                             da.name, da.bus->name);
3037                 return;
3038         }
3039
3040         remove_invalid_ports();
3041
3042         printf("Device %s is detached\n", identifier);
3043         printf("Now total ports is %d\n", nb_ports);
3044         printf("Done\n");
3045 }
3046
3047 void
3048 pmd_test_exit(void)
3049 {
3050         portid_t pt_id;
3051         unsigned int i;
3052         int ret;
3053
3054         if (test_done == 0)
3055                 stop_packet_forwarding();
3056
3057         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3058                 if (mempools[i]) {
3059                         if (mp_alloc_type == MP_ALLOC_ANON)
3060                                 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3061                                                      NULL);
3062                 }
3063         }
3064         if (ports != NULL) {
3065                 no_link_check = 1;
3066                 RTE_ETH_FOREACH_DEV(pt_id) {
3067                         printf("\nStopping port %d...\n", pt_id);
3068                         fflush(stdout);
3069                         stop_port(pt_id);
3070                 }
3071                 RTE_ETH_FOREACH_DEV(pt_id) {
3072                         printf("\nShutting down port %d...\n", pt_id);
3073                         fflush(stdout);
3074                         close_port(pt_id);
3075                 }
3076         }
3077
3078         if (hot_plug) {
3079                 ret = rte_dev_event_monitor_stop();
3080                 if (ret) {
3081                         RTE_LOG(ERR, EAL,
3082                                 "fail to stop device event monitor.");
3083                         return;
3084                 }
3085
3086                 ret = rte_dev_event_callback_unregister(NULL,
3087                         dev_event_callback, NULL);
3088                 if (ret < 0) {
3089                         RTE_LOG(ERR, EAL,
3090                                 "fail to unregister device event callback.\n");
3091                         return;
3092                 }
3093
3094                 ret = rte_dev_hotplug_handle_disable();
3095                 if (ret) {
3096                         RTE_LOG(ERR, EAL,
3097                                 "fail to disable hotplug handling.\n");
3098                         return;
3099                 }
3100         }
3101         for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3102                 if (mempools[i])
3103                         rte_mempool_free(mempools[i]);
3104         }
3105
3106         printf("\nBye...\n");
3107 }
3108
3109 typedef void (*cmd_func_t)(void);
3110 struct pmd_test_command {
3111         const char *cmd_name;
3112         cmd_func_t cmd_func;
3113 };
3114
3115 /* Check the link status of all ports in up to 9s, and print them finally */
3116 static void
3117 check_all_ports_link_status(uint32_t port_mask)
3118 {
3119 #define CHECK_INTERVAL 100 /* 100ms */
3120 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3121         portid_t portid;
3122         uint8_t count, all_ports_up, print_flag = 0;
3123         struct rte_eth_link link;
3124         int ret;
3125         char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3126
3127         printf("Checking link statuses...\n");
3128         fflush(stdout);
3129         for (count = 0; count <= MAX_CHECK_TIME; count++) {
3130                 all_ports_up = 1;
3131                 RTE_ETH_FOREACH_DEV(portid) {
3132                         if ((port_mask & (1 << portid)) == 0)
3133                                 continue;
3134                         memset(&link, 0, sizeof(link));
3135                         ret = rte_eth_link_get_nowait(portid, &link);
3136                         if (ret < 0) {
3137                                 all_ports_up = 0;
3138                                 if (print_flag == 1)
3139                                         printf("Port %u link get failed: %s\n",
3140                                                 portid, rte_strerror(-ret));
3141                                 continue;
3142                         }
3143                         /* print link status if flag set */
3144                         if (print_flag == 1) {
3145                                 rte_eth_link_to_str(link_status,
3146                                         sizeof(link_status), &link);
3147                                 printf("Port %d %s\n", portid, link_status);
3148                                 continue;
3149                         }
3150                         /* clear all_ports_up flag if any link down */
3151                         if (link.link_status == ETH_LINK_DOWN) {
3152                                 all_ports_up = 0;
3153                                 break;
3154                         }
3155                 }
3156                 /* after finally printing all link status, get out */
3157                 if (print_flag == 1)
3158                         break;
3159
3160                 if (all_ports_up == 0) {
3161                         fflush(stdout);
3162                         rte_delay_ms(CHECK_INTERVAL);
3163                 }
3164
3165                 /* set the print_flag if all ports up or timeout */
3166                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3167                         print_flag = 1;
3168                 }
3169
3170                 if (lsc_interrupt)
3171                         break;
3172         }
3173 }
3174
3175 static void
3176 rmv_port_callback(void *arg)
3177 {
3178         int need_to_start = 0;
3179         int org_no_link_check = no_link_check;
3180         portid_t port_id = (intptr_t)arg;
3181         struct rte_device *dev;
3182
3183         RTE_ETH_VALID_PORTID_OR_RET(port_id);
3184
3185         if (!test_done && port_is_forwarding(port_id)) {
3186                 need_to_start = 1;
3187                 stop_packet_forwarding();
3188         }
3189         no_link_check = 1;
3190         stop_port(port_id);
3191         no_link_check = org_no_link_check;
3192
3193         /* Save rte_device pointer before closing ethdev port */
3194         dev = rte_eth_devices[port_id].device;
3195         close_port(port_id);
3196         detach_device(dev); /* might be already removed or have more ports */
3197
3198         if (need_to_start)
3199                 start_packet_forwarding(0);
3200 }
3201
3202 /* This function is used by the interrupt thread */
3203 static int
3204 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3205                   void *ret_param)
3206 {
3207         RTE_SET_USED(param);
3208         RTE_SET_USED(ret_param);
3209
3210         if (type >= RTE_ETH_EVENT_MAX) {
3211                 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3212                         port_id, __func__, type);
3213                 fflush(stderr);
3214         } else if (event_print_mask & (UINT32_C(1) << type)) {
3215                 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3216                         eth_event_desc[type]);
3217                 fflush(stdout);
3218         }
3219
3220         switch (type) {
3221         case RTE_ETH_EVENT_NEW:
3222                 ports[port_id].need_setup = 1;
3223                 ports[port_id].port_status = RTE_PORT_HANDLING;
3224                 break;
3225         case RTE_ETH_EVENT_INTR_RMV:
3226                 if (port_id_is_invalid(port_id, DISABLED_WARN))
3227                         break;
3228                 if (rte_eal_alarm_set(100000,
3229                                 rmv_port_callback, (void *)(intptr_t)port_id))
3230                         fprintf(stderr, "Could not set up deferred device removal\n");
3231                 break;
3232         case RTE_ETH_EVENT_DESTROY:
3233                 ports[port_id].port_status = RTE_PORT_CLOSED;
3234                 printf("Port %u is closed\n", port_id);
3235                 break;
3236         default:
3237                 break;
3238         }
3239         return 0;
3240 }
3241
3242 static int
3243 register_eth_event_callback(void)
3244 {
3245         int ret;
3246         enum rte_eth_event_type event;
3247
3248         for (event = RTE_ETH_EVENT_UNKNOWN;
3249                         event < RTE_ETH_EVENT_MAX; event++) {
3250                 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3251                                 event,
3252                                 eth_event_callback,
3253                                 NULL);
3254                 if (ret != 0) {
3255                         TESTPMD_LOG(ERR, "Failed to register callback for "
3256                                         "%s event\n", eth_event_desc[event]);
3257                         return -1;
3258                 }
3259         }
3260
3261         return 0;
3262 }
3263
3264 /* This function is used by the interrupt thread */
3265 static void
3266 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3267                              __rte_unused void *arg)
3268 {
3269         uint16_t port_id;
3270         int ret;
3271
3272         if (type >= RTE_DEV_EVENT_MAX) {
3273                 fprintf(stderr, "%s called upon invalid event %d\n",
3274                         __func__, type);
3275                 fflush(stderr);
3276         }
3277
3278         switch (type) {
3279         case RTE_DEV_EVENT_REMOVE:
3280                 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3281                         device_name);
3282                 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3283                 if (ret) {
3284                         RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3285                                 device_name);
3286                         return;
3287                 }
3288                 /*
3289                  * Because the user's callback is invoked in eal interrupt
3290                  * callback, the interrupt callback need to be finished before
3291                  * it can be unregistered when detaching device. So finish
3292                  * callback soon and use a deferred removal to detach device
3293                  * is need. It is a workaround, once the device detaching be
3294                  * moved into the eal in the future, the deferred removal could
3295                  * be deleted.
3296                  */
3297                 if (rte_eal_alarm_set(100000,
3298                                 rmv_port_callback, (void *)(intptr_t)port_id))
3299                         RTE_LOG(ERR, EAL,
3300                                 "Could not set up deferred device removal\n");
3301                 break;
3302         case RTE_DEV_EVENT_ADD:
3303                 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3304                         device_name);
3305                 /* TODO: After finish kernel driver binding,
3306                  * begin to attach port.
3307                  */
3308                 break;
3309         default:
3310                 break;
3311         }
3312 }
3313
3314 static void
3315 rxtx_port_config(struct rte_port *port)
3316 {
3317         uint16_t qid;
3318         uint64_t offloads;
3319
3320         for (qid = 0; qid < nb_rxq; qid++) {
3321                 offloads = port->rx_conf[qid].offloads;
3322                 port->rx_conf[qid] = port->dev_info.default_rxconf;
3323                 if (offloads != 0)
3324                         port->rx_conf[qid].offloads = offloads;
3325
3326                 /* Check if any Rx parameters have been passed */
3327                 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3328                         port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3329
3330                 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3331                         port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3332
3333                 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3334                         port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3335
3336                 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3337                         port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3338
3339                 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3340                         port->rx_conf[qid].rx_drop_en = rx_drop_en;
3341
3342                 port->nb_rx_desc[qid] = nb_rxd;
3343         }
3344
3345         for (qid = 0; qid < nb_txq; qid++) {
3346                 offloads = port->tx_conf[qid].offloads;
3347                 port->tx_conf[qid] = port->dev_info.default_txconf;
3348                 if (offloads != 0)
3349                         port->tx_conf[qid].offloads = offloads;
3350
3351                 /* Check if any Tx parameters have been passed */
3352                 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3353                         port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3354
3355                 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3356                         port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3357
3358                 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3359                         port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3360
3361                 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3362                         port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3363
3364                 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3365                         port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3366
3367                 port->nb_tx_desc[qid] = nb_txd;
3368         }
3369 }
3370
3371 /*
3372  * Helper function to arrange max_rx_pktlen value and JUMBO_FRAME offload,
3373  * MTU is also aligned if JUMBO_FRAME offload is not set.
3374  *
3375  * port->dev_info should be set before calling this function.
3376  *
3377  * return 0 on success, negative on error
3378  */
3379 int
3380 update_jumbo_frame_offload(portid_t portid)
3381 {
3382         struct rte_port *port = &ports[portid];
3383         uint32_t eth_overhead;
3384         uint64_t rx_offloads;
3385         int ret;
3386         bool on;
3387
3388         /* Update the max_rx_pkt_len to have MTU as RTE_ETHER_MTU */
3389         if (port->dev_info.max_mtu != UINT16_MAX &&
3390             port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3391                 eth_overhead = port->dev_info.max_rx_pktlen -
3392                                 port->dev_info.max_mtu;
3393         else
3394                 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3395
3396         rx_offloads = port->dev_conf.rxmode.offloads;
3397
3398         /* Default config value is 0 to use PMD specific overhead */
3399         if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3400                 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3401
3402         if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3403                 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3404                 on = false;
3405         } else {
3406                 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3407                         printf("Frame size (%u) is not supported by port %u\n",
3408                                 port->dev_conf.rxmode.max_rx_pkt_len,
3409                                 portid);
3410                         return -1;
3411                 }
3412                 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3413                 on = true;
3414         }
3415
3416         if (rx_offloads != port->dev_conf.rxmode.offloads) {
3417                 uint16_t qid;
3418
3419                 port->dev_conf.rxmode.offloads = rx_offloads;
3420
3421                 /* Apply JUMBO_FRAME offload configuration to Rx queue(s) */
3422                 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3423                         if (on)
3424                                 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3425                         else
3426                                 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3427                 }
3428         }
3429
3430         /* If JUMBO_FRAME is set MTU conversion done by ethdev layer,
3431          * if unset do it here
3432          */
3433         if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3434                 ret = rte_eth_dev_set_mtu(portid,
3435                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3436                 if (ret)
3437                         printf("Failed to set MTU to %u for port %u\n",
3438                                 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3439                                 portid);
3440         }
3441
3442         return 0;
3443 }
3444
3445 void
3446 init_port_config(void)
3447 {
3448         portid_t pid;
3449         struct rte_port *port;
3450         int ret;
3451
3452         RTE_ETH_FOREACH_DEV(pid) {
3453                 port = &ports[pid];
3454                 port->dev_conf.fdir_conf = fdir_conf;
3455
3456                 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3457                 if (ret != 0)
3458                         return;
3459
3460                 if (nb_rxq > 1) {
3461                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3462                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3463                                 rss_hf & port->dev_info.flow_type_rss_offloads;
3464                 } else {
3465                         port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3466                         port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3467                 }
3468
3469                 if (port->dcb_flag == 0) {
3470                         if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3471                                 port->dev_conf.rxmode.mq_mode =
3472                                         (enum rte_eth_rx_mq_mode)
3473                                                 (rx_mq_mode & ETH_MQ_RX_RSS);
3474                         else
3475                                 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3476                 }
3477
3478                 rxtx_port_config(port);
3479
3480                 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3481                 if (ret != 0)
3482                         return;
3483
3484 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3485                 rte_pmd_ixgbe_bypass_init(pid);
3486 #endif
3487
3488                 if (lsc_interrupt &&
3489                     (rte_eth_devices[pid].data->dev_flags &
3490                      RTE_ETH_DEV_INTR_LSC))
3491                         port->dev_conf.intr_conf.lsc = 1;
3492                 if (rmv_interrupt &&
3493                     (rte_eth_devices[pid].data->dev_flags &
3494                      RTE_ETH_DEV_INTR_RMV))
3495                         port->dev_conf.intr_conf.rmv = 1;
3496         }
3497 }
3498
3499 void set_port_slave_flag(portid_t slave_pid)
3500 {
3501         struct rte_port *port;
3502
3503         port = &ports[slave_pid];
3504         port->slave_flag = 1;
3505 }
3506
3507 void clear_port_slave_flag(portid_t slave_pid)
3508 {
3509         struct rte_port *port;
3510
3511         port = &ports[slave_pid];
3512         port->slave_flag = 0;
3513 }
3514
3515 uint8_t port_is_bonding_slave(portid_t slave_pid)
3516 {
3517         struct rte_port *port;
3518
3519         port = &ports[slave_pid];
3520         if ((rte_eth_devices[slave_pid].data->dev_flags &
3521             RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3522                 return 1;
3523         return 0;
3524 }
3525
3526 const uint16_t vlan_tags[] = {
3527                 0,  1,  2,  3,  4,  5,  6,  7,
3528                 8,  9, 10, 11,  12, 13, 14, 15,
3529                 16, 17, 18, 19, 20, 21, 22, 23,
3530                 24, 25, 26, 27, 28, 29, 30, 31
3531 };
3532
3533 static  int
3534 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3535                  enum dcb_mode_enable dcb_mode,
3536                  enum rte_eth_nb_tcs num_tcs,
3537                  uint8_t pfc_en)
3538 {
3539         uint8_t i;
3540         int32_t rc;
3541         struct rte_eth_rss_conf rss_conf;
3542
3543         /*
3544          * Builds up the correct configuration for dcb+vt based on the vlan tags array
3545          * given above, and the number of traffic classes available for use.
3546          */
3547         if (dcb_mode == DCB_VT_ENABLED) {
3548                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3549                                 &eth_conf->rx_adv_conf.vmdq_dcb_conf;
3550                 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3551                                 &eth_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3552
3553                 /* VMDQ+DCB RX and TX configurations */
3554                 vmdq_rx_conf->enable_default_pool = 0;
3555                 vmdq_rx_conf->default_pool = 0;
3556                 vmdq_rx_conf->nb_queue_pools =
3557                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3558                 vmdq_tx_conf->nb_queue_pools =
3559                         (num_tcs ==  ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3560
3561                 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3562                 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3563                         vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3564                         vmdq_rx_conf->pool_map[i].pools =
3565                                 1 << (i % vmdq_rx_conf->nb_queue_pools);
3566                 }
3567                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3568                         vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3569                         vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3570                 }
3571
3572                 /* set DCB mode of RX and TX of multiple queues */
3573                 eth_conf->rxmode.mq_mode =
3574                                 (enum rte_eth_rx_mq_mode)
3575                                         (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3576                 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3577         } else {
3578                 struct rte_eth_dcb_rx_conf *rx_conf =
3579                                 &eth_conf->rx_adv_conf.dcb_rx_conf;
3580                 struct rte_eth_dcb_tx_conf *tx_conf =
3581                                 &eth_conf->tx_adv_conf.dcb_tx_conf;
3582
3583                 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3584
3585                 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3586                 if (rc != 0)
3587                         return rc;
3588
3589                 rx_conf->nb_tcs = num_tcs;
3590                 tx_conf->nb_tcs = num_tcs;
3591
3592                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3593                         rx_conf->dcb_tc[i] = i % num_tcs;
3594                         tx_conf->dcb_tc[i] = i % num_tcs;
3595                 }
3596
3597                 eth_conf->rxmode.mq_mode =
3598                                 (enum rte_eth_rx_mq_mode)
3599                                         (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3600                 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3601                 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3602         }
3603
3604         if (pfc_en)
3605                 eth_conf->dcb_capability_en =
3606                                 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3607         else
3608                 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3609
3610         return 0;
3611 }
3612
3613 int
3614 init_port_dcb_config(portid_t pid,
3615                      enum dcb_mode_enable dcb_mode,
3616                      enum rte_eth_nb_tcs num_tcs,
3617                      uint8_t pfc_en)
3618 {
3619         struct rte_eth_conf port_conf;
3620         struct rte_port *rte_port;
3621         int retval;
3622         uint16_t i;
3623
3624         rte_port = &ports[pid];
3625
3626         memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3627         /* Enter DCB configuration status */
3628         dcb_config = 1;
3629
3630         port_conf.rxmode = rte_port->dev_conf.rxmode;
3631         port_conf.txmode = rte_port->dev_conf.txmode;
3632
3633         /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3634         retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3635         if (retval < 0)
3636                 return retval;
3637         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3638
3639         /* re-configure the device . */
3640         retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3641         if (retval < 0)
3642                 return retval;
3643
3644         retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3645         if (retval != 0)
3646                 return retval;
3647
3648         /* If dev_info.vmdq_pool_base is greater than 0,
3649          * the queue id of vmdq pools is started after pf queues.
3650          */
3651         if (dcb_mode == DCB_VT_ENABLED &&
3652             rte_port->dev_info.vmdq_pool_base > 0) {
3653                 printf("VMDQ_DCB multi-queue mode is nonsensical"
3654                         " for port %d.", pid);
3655                 return -1;
3656         }
3657
3658         /* Assume the ports in testpmd have the same dcb capability
3659          * and has the same number of rxq and txq in dcb mode
3660          */
3661         if (dcb_mode == DCB_VT_ENABLED) {
3662                 if (rte_port->dev_info.max_vfs > 0) {
3663                         nb_rxq = rte_port->dev_info.nb_rx_queues;
3664                         nb_txq = rte_port->dev_info.nb_tx_queues;
3665                 } else {
3666                         nb_rxq = rte_port->dev_info.max_rx_queues;
3667                         nb_txq = rte_port->dev_info.max_tx_queues;
3668                 }
3669         } else {
3670                 /*if vt is disabled, use all pf queues */
3671                 if (rte_port->dev_info.vmdq_pool_base == 0) {
3672                         nb_rxq = rte_port->dev_info.max_rx_queues;
3673                         nb_txq = rte_port->dev_info.max_tx_queues;
3674                 } else {
3675                         nb_rxq = (queueid_t)num_tcs;
3676                         nb_txq = (queueid_t)num_tcs;
3677
3678                 }
3679         }
3680         rx_free_thresh = 64;
3681
3682         memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3683
3684         rxtx_port_config(rte_port);
3685         /* VLAN filter */
3686         rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3687         for (i = 0; i < RTE_DIM(vlan_tags); i++)
3688                 rx_vft_set(pid, vlan_tags[i], 1);
3689
3690         retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3691         if (retval != 0)
3692                 return retval;
3693
3694         rte_port->dcb_flag = 1;
3695
3696         return 0;
3697 }
3698
3699 static void
3700 init_port(void)
3701 {
3702         int i;
3703
3704         /* Configuration of Ethernet ports. */
3705         ports = rte_zmalloc("testpmd: ports",
3706                             sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3707                             RTE_CACHE_LINE_SIZE);
3708         if (ports == NULL) {
3709                 rte_exit(EXIT_FAILURE,
3710                                 "rte_zmalloc(%d struct rte_port) failed\n",
3711                                 RTE_MAX_ETHPORTS);
3712         }
3713         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3714                 LIST_INIT(&ports[i].flow_tunnel_list);
3715         /* Initialize ports NUMA structures */
3716         memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3717         memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3718         memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3719 }
3720
3721 static void
3722 force_quit(void)
3723 {
3724         pmd_test_exit();
3725         prompt_exit();
3726 }
3727
3728 static void
3729 print_stats(void)
3730 {
3731         uint8_t i;
3732         const char clr[] = { 27, '[', '2', 'J', '\0' };
3733         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3734
3735         /* Clear screen and move to top left */
3736         printf("%s%s", clr, top_left);
3737
3738         printf("\nPort statistics ====================================");
3739         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3740                 nic_stats_display(fwd_ports_ids[i]);
3741
3742         fflush(stdout);
3743 }
3744
3745 static void
3746 signal_handler(int signum)
3747 {
3748         if (signum == SIGINT || signum == SIGTERM) {
3749                 printf("\nSignal %d received, preparing to exit...\n",
3750                                 signum);
3751 #ifdef RTE_LIB_PDUMP
3752                 /* uninitialize packet capture framework */
3753                 rte_pdump_uninit();
3754 #endif
3755 #ifdef RTE_LIB_LATENCYSTATS
3756                 if (latencystats_enabled != 0)
3757                         rte_latencystats_uninit();
3758 #endif
3759                 force_quit();
3760                 /* Set flag to indicate the force termination. */
3761                 f_quit = 1;
3762                 /* exit with the expected status */
3763                 signal(signum, SIG_DFL);
3764                 kill(getpid(), signum);
3765         }
3766 }
3767
3768 int
3769 main(int argc, char** argv)
3770 {
3771         int diag;
3772         portid_t port_id;
3773         uint16_t count;
3774         int ret;
3775
3776         signal(SIGINT, signal_handler);
3777         signal(SIGTERM, signal_handler);
3778
3779         testpmd_logtype = rte_log_register("testpmd");
3780         if (testpmd_logtype < 0)
3781                 rte_exit(EXIT_FAILURE, "Cannot register log type");
3782         rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3783
3784         diag = rte_eal_init(argc, argv);
3785         if (diag < 0)
3786                 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3787                          rte_strerror(rte_errno));
3788
3789         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3790                 rte_exit(EXIT_FAILURE,
3791                          "Secondary process type not supported.\n");
3792
3793         ret = register_eth_event_callback();
3794         if (ret != 0)
3795                 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3796
3797 #ifdef RTE_LIB_PDUMP
3798         /* initialize packet capture framework */
3799         rte_pdump_init();
3800 #endif
3801
3802         count = 0;
3803         RTE_ETH_FOREACH_DEV(port_id) {
3804                 ports_ids[count] = port_id;
3805                 count++;
3806         }
3807         nb_ports = (portid_t) count;
3808         if (nb_ports == 0)
3809                 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3810
3811         /* allocate port structures, and init them */
3812         init_port();
3813
3814         set_def_fwd_config();
3815         if (nb_lcores == 0)
3816                 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3817                          "Check the core mask argument\n");
3818
3819         /* Bitrate/latency stats disabled by default */
3820 #ifdef RTE_LIB_BITRATESTATS
3821         bitrate_enabled = 0;
3822 #endif
3823 #ifdef RTE_LIB_LATENCYSTATS
3824         latencystats_enabled = 0;
3825 #endif
3826
3827         /* on FreeBSD, mlockall() is disabled by default */
3828 #ifdef RTE_EXEC_ENV_FREEBSD
3829         do_mlockall = 0;
3830 #else
3831         do_mlockall = 1;
3832 #endif
3833
3834         argc -= diag;
3835         argv += diag;
3836         if (argc > 1)
3837                 launch_args_parse(argc, argv);
3838
3839         if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3840                 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3841                         strerror(errno));
3842         }
3843
3844         if (tx_first && interactive)
3845                 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3846                                 "interactive mode.\n");
3847
3848         if (tx_first && lsc_interrupt) {
3849                 printf("Warning: lsc_interrupt needs to be off when "
3850                                 " using tx_first. Disabling.\n");
3851                 lsc_interrupt = 0;
3852         }
3853
3854         if (!nb_rxq && !nb_txq)
3855                 printf("Warning: Either rx or tx queues should be non-zero\n");
3856
3857         if (nb_rxq > 1 && nb_rxq > nb_txq)
3858                 printf("Warning: nb_rxq=%d enables RSS configuration, "
3859                        "but nb_txq=%d will prevent to fully test it.\n",
3860                        nb_rxq, nb_txq);
3861
3862         init_config();
3863
3864         if (hot_plug) {
3865                 ret = rte_dev_hotplug_handle_enable();
3866                 if (ret) {
3867                         RTE_LOG(ERR, EAL,
3868                                 "fail to enable hotplug handling.");
3869                         return -1;
3870                 }
3871
3872                 ret = rte_dev_event_monitor_start();
3873                 if (ret) {
3874                         RTE_LOG(ERR, EAL,
3875                                 "fail to start device event monitoring.");
3876                         return -1;
3877                 }
3878
3879                 ret = rte_dev_event_callback_register(NULL,
3880                         dev_event_callback, NULL);
3881                 if (ret) {
3882                         RTE_LOG(ERR, EAL,
3883                                 "fail  to register device event callback\n");
3884                         return -1;
3885                 }
3886         }
3887
3888         if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3889                 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3890
3891         /* set all ports to promiscuous mode by default */
3892         RTE_ETH_FOREACH_DEV(port_id) {
3893                 ret = rte_eth_promiscuous_enable(port_id);
3894                 if (ret != 0)
3895                         printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3896                                 port_id, rte_strerror(-ret));
3897         }
3898
3899         /* Init metrics library */
3900         rte_metrics_init(rte_socket_id());
3901
3902 #ifdef RTE_LIB_LATENCYSTATS
3903         if (latencystats_enabled != 0) {
3904                 int ret = rte_latencystats_init(1, NULL);
3905                 if (ret)
3906                         printf("Warning: latencystats init()"
3907                                 " returned error %d\n", ret);
3908                 printf("Latencystats running on lcore %d\n",
3909                         latencystats_lcore_id);
3910         }
3911 #endif
3912
3913         /* Setup bitrate stats */
3914 #ifdef RTE_LIB_BITRATESTATS
3915         if (bitrate_enabled != 0) {
3916                 bitrate_data = rte_stats_bitrate_create();
3917                 if (bitrate_data == NULL)
3918                         rte_exit(EXIT_FAILURE,
3919                                 "Could not allocate bitrate data.\n");
3920                 rte_stats_bitrate_reg(bitrate_data);
3921         }
3922 #endif
3923
3924 #ifdef RTE_LIB_CMDLINE
3925         if (strlen(cmdline_filename) != 0)
3926                 cmdline_read_from_file(cmdline_filename);
3927
3928         if (interactive == 1) {
3929                 if (auto_start) {
3930                         printf("Start automatic packet forwarding\n");
3931                         start_packet_forwarding(0);
3932                 }
3933                 prompt();
3934                 pmd_test_exit();
3935         } else
3936 #endif
3937         {
3938                 char c;
3939                 int rc;
3940
3941                 f_quit = 0;
3942
3943                 printf("No commandline core given, start packet forwarding\n");
3944                 start_packet_forwarding(tx_first);
3945                 if (stats_period != 0) {
3946                         uint64_t prev_time = 0, cur_time, diff_time = 0;
3947                         uint64_t timer_period;
3948
3949                         /* Convert to number of cycles */
3950                         timer_period = stats_period * rte_get_timer_hz();
3951
3952                         while (f_quit == 0) {
3953                                 cur_time = rte_get_timer_cycles();
3954                                 diff_time += cur_time - prev_time;
3955
3956                                 if (diff_time >= timer_period) {
3957                                         print_stats();
3958                                         /* Reset the timer */
3959                                         diff_time = 0;
3960                                 }
3961                                 /* Sleep to avoid unnecessary checks */
3962                                 prev_time = cur_time;
3963                                 sleep(1);
3964                         }
3965                 }
3966
3967                 printf("Press enter to exit\n");
3968                 rc = read(0, &c, 1);
3969                 pmd_test_exit();
3970                 if (rc < 0)
3971                         return 1;
3972         }
3973
3974         ret = rte_eal_cleanup();
3975         if (ret != 0)
3976                 rte_exit(EXIT_FAILURE,
3977                          "EAL cleanup failed: %s\n", strerror(-ret));
3978
3979         return EXIT_SUCCESS;
3980 }